Esempio n. 1
0
def calibrate_file(term, log, cl_params):
    """Calibrate a single SDFITS file

       Actual calibration is done in calibrate_win_feed_pol(),
       which is called by calibrate_maps().

    """

    # Instantiate a SdFits object for I/O and interpreting the
    #  contents of the index file
    sdf = SdFits()

    # generate a name for the index file based on the name of the
    #  raw SDFITS file.  The index file simple has a different extension
    indexfile = sdf.nameIndexFile(cl_params.infilename)
    try:
        # create a structure that lists the raw SDFITS rows for
        #  each scan/window/feed/polarization
        row_list = sdf.parseSdfitsIndex(indexfile)
    except IOError:
        log.doMessage('ERR', 'Could not open index file', indexfile)
        sys.exit()

    # if there are no mapscans set at the command line, the user
    #  probably wants the pipeline to find maps in the input file
    if not cl_params.mapscans:
        if cl_params.refscans:
            log.doMessage('WARN', 'Referene scan(s) given without map scans, '
                          'ignoring reference scan settings.')
        cl_params.refscans = []
         # this is where we try to guess the mapping blocks in the
        #  input file.  A powition-switched mapping block is defined
        #  as a reference scan, followed by mapping scans, optionally
        #  followed by another reference scan.  The returned structure
        #  is a list of tuples of (reference1, mapscans, reference2)
        maps = sdf.find_maps(indexfile)

        if maps:
            log.doMessage('INFO', 'Found', len(maps), 'map(s).')
        else:
            log.doMessage('ERR', 'No scans specified and found no '
                          'position-switched maps.')
            sys.exit()

        # calibrate each map found in the input file
        for map_number, map_params in enumerate(maps):
            cl_params = set_map_scans(cl_params, map_params)
            log.doMessage('INFO', 'Processing map:', str(map_number+1), 'of',
                          len(maps))
            calibrated_maps = calibrate_maps(log, cl_params, row_list, term)
    else:
        # calibrate the map defined by the user
        calibrated_maps = calibrate_maps(log, cl_params, row_list, term)

    return calibrated_maps
Esempio n. 2
0
def runPipeline():

    # the blessings.Terminal object is used for printing to the screen
    terminal = blessings.Terminal()
    term = terminal

    # create instance of CommandLine object to parse input, then
    # parse all the input parameters and store them as attributes in
    # cl_params structure
    cl = commandline.CommandLine()
    cl_params = cl.read(sys)

    # create a directory for storing log files
    mkdir_p('log')

    # instantiate a log object for log files and screen output
    log = Logging(cl_params, 'pipeline')

    # print a command summary to log file/terminal
    command_summary(cl_params, term, log)

    if not cl_params.imagingoff:

        # instantiate an Imaging object
        imag = Imaging.Imaging()

    else:
        imag = None

    log.doMessage('INFO', '{t.underline}Start '
                  'calibration.{t.normal}'.format(t=term))

    # calibrated_maps will contain a list of tuples each with a
    # MappingPipeline instance, window id, feed id and polarization id
    calibrated_maps = []

    # check to see if the infile parameter is a directory.  This is the
    # default case for vegas data processing.  If we have a directory of
    # files, we calibrate each input file separately, then perform the
    # imaging step on all the calibrated outputs.
    if os.path.isdir(cl_params.infilename):
        log.doMessage('INFO', 'Infile name is a directory')
        input_directory = cl_params.infilename

        # Instantiate a SdFits object for I/O and interpreting the
        #  contents of the index file
        sdf = SdFits()

        # generate a name for the index file based on the name of the
        #  raw SDFITS file.  The index file simply has a different extension
        directory_name = os.path.basename(cl_params.infilename.rstrip('/'))
        indexfile = cl_params.infilename + '/' + directory_name + '.index'
        try:
            # create a structure that lists the raw SDFITS rows for
            #  each scan/window/feed/polarization
            row_list, summary = sdf.parseSdfitsIndex(indexfile, cl_params.mapscans)
        except IOError:
            log.doMessage('ERR', 'Could not open index file', indexfile)
            sys.exit()

        quitcal = False
        if cl_params.window and set(cl_params.window).isdisjoint(set(row_list.windows())):
            log.doMessage('ERR', 'no given WINDOW(S)', cl_params.window, 'in dataset')
            quitcal = True
        if cl_params.feed and set(cl_params.feed).isdisjoint(set(row_list.feeds())):
            log.doMessage('ERR', 'no given FEED(S)', cl_params.feed, 'in dataset')
            quitcal = True
        if cl_params.pol and set(cl_params.pol).isdisjoint(set(row_list.pols())):
            log.doMessage('ERR', 'no given POL(S)', cl_params.pol, 'in dataset')
            quitcal = True
        if cl_params.mapscans and set(cl_params.mapscans).isdisjoint(set(row_list.scans())):
            log.doMessage('ERR', 'no given MAPSCAN(S)', cl_params.mapscans, 'in dataset')
            quitcal = True
        if cl_params.refscans and set(cl_params.refscans).isdisjoint(set(row_list.scans())):
            log.doMessage('ERR', 'no given REFSCAN(S)', cl_params.refscans, 'in dataset')
            quitcal = True

        if quitcal:
            sys.exit(12)

        # calibrate one raw SDFITS file at a time
        for infilename in glob.glob(input_directory + '/' +
                                    os.path.basename(input_directory) +
                                    '*.fits'):
            log.doMessage('DBG', 'Attempting to calibrate', os.path.basename(infilename).rstrip('.fits'))
            # change the infilename in the params structure to the
            #  current infile in the directory for each iteration
            cl_params.infilename = infilename

            # copy the cl_params structure so we can modify it during calibration
            # for each seperate file.
            commandline_options = copy.deepcopy(cl_params)
            calibrated_maps_this_file = calibrate_file(term, log, commandline_options)
            if calibrated_maps_this_file:
                calibrated_maps.extend(calibrated_maps_this_file)
    else:
        commandline_options = copy.deepcopy(cl_params)
        calibrated_maps_this_file = calibrate_file(term, log, commandline_options)
        if calibrated_maps_this_file:
            calibrated_maps.extend(calibrated_maps_this_file)

    if not calibrated_maps:
        log.doMessage('ERR', 'No calibrated spectra.  Check inputs and try again')
        sys.exit(-1)

    # if we are doing imaging
    if not cl_params.imagingoff:

        # image all the calibrated maps
        import itertools
        calibrated_maps = list(itertools.chain(*calibrated_maps))

        imag.run(log, term, cl_params, calibrated_maps)

    sys.stdout.write('\n')
Esempio n. 3
0
def calibrate_file(term, log, command_options):
    """Calibrate a single SDFITS file

       Actual calibration is done in calibrate_win_feed_pol(),
       which is called by calibrate_maps().

    """

    # Instantiate a SdFits object for I/O and interpreting the
    #  contents of the index file
    sdf = SdFits()

    # generate a name for the index file based on the name of the
    #  raw SDFITS file.  The index file simply has a different extension
    indexfile = sdf.nameIndexFile(command_options.infilename)
    try:
        # create a structure that lists the raw SDFITS rows for
        #  each scan/window/feed/polarization
        row_list, summary = sdf.parseSdfitsIndex(indexfile, command_options.mapscans)
    except IOError:
        log.doMessage('ERR', 'Could not open index file', indexfile)
        sys.exit()

    log.doMessage('INFO', indexfile)
    log.doMessage('INFO', '    ', len(summary['WINDOWS']), 'spectral window(s)')
    for (win, freq) in sorted(summary['WINDOWS']):
        log.doMessage('INFO', '       {win}: {freq:.4f} GHz'.format(win=win, freq=freq))
    log.doMessage('INFO', '    ', len(summary['FEEDS']), 'feed(s):', ', '.join(sorted(summary['FEEDS'])))

    proceed_with_calibration = True

    # check for presence of pol(s) in dataset before attempting calibration
    if command_options.pol:
        if set(command_options.pol).isdisjoint(set(row_list.pols())):
            log.doMessage('WARN', 'POL', command_options.pol, 'not in', os.path.basename(indexfile))
            proceed_with_calibration = False
        else:
            requested_pols = command_options.pol
            command_options.pol = list(set(command_options.pol).intersection(set(row_list.pols())))
            pol_diff = set(requested_pols).difference(set(command_options.pol))
            if pol_diff:
                log.doMessage('WARN', 'POL', list(pol_diff), 'not in', os.path.basename(indexfile))

    # check for presence of feed(s) in dataset before attempting calibration
    if command_options.feed:
        if set(command_options.feed).isdisjoint(set(row_list.feeds())):
            log.doMessage('WARN', 'FEED', command_options.feed, 'not in', os.path.basename(indexfile))
            proceed_with_calibration = False
        else:
            requested_feeds = command_options.feed
            command_options.feed = list(set(command_options.feed).intersection(set(row_list.feeds())))
            feed_diff = set(requested_feeds).difference(set(command_options.feed))
            if feed_diff:
                log.doMessage('WARN', 'FEED', list(feed_diff), 'not in', os.path.basename(indexfile))

    # check for presence of window(s) in dataset before attempting calibration
    if command_options.window:
        if set(command_options.window).isdisjoint(set(row_list.windows())):
            log.doMessage('DBG', 'WINDOW', command_options.window, 'not in', os.path.basename(indexfile))
            proceed_with_calibration = False
        else:
            requested_windows = command_options.window
            command_options.window = list(set(command_options.window).intersection(set(row_list.windows())))
            window_diff = set(requested_windows).difference(set(command_options.window))
            if window_diff:
                log.doMessage('DBG', 'WINDOW', list(window_diff), 'not in', os.path.basename(indexfile))

    if proceed_with_calibration is False:
        return None

    calibrated_maps = []
    # if there are no mapscans set at the command line, the user
    #  probably wants the pipeline to find maps in the input file
    if not command_options.mapscans:
        if command_options.refscans:
            log.doMessage('WARN', 'Referene scan(s) given without map scans, '
                          'ignoring reference scan settings.')
        command_options.refscans = []
        # this is where we try to guess the mapping blocks in the
        #  input file.  A powition-switched mapping block is defined
        #  as a reference scan, followed by mapping scans, optionally
        #  followed by another reference scan.  The returned structure
        #  is a list of tuples of (reference1, mapscans, reference2)
        maps = sdf.find_maps(indexfile)

        if maps:
            log.doMessage('INFO', 'Found', len(maps), 'map(s).')
        else:
            log.doMessage('ERR', 'No scans specified and found no '
                          'position-switched maps.')
            sys.exit()

        # calibrate each map found in the input file
        for map_number, map_params in enumerate(maps):
            cmd_options = set_map_scans(command_options, map_params)
            log.doMessage('INFO', '\nProcessing map:', str(map_number+1), 'of',
                          len(maps))
            calibrated_maps.append(calibrate_maps(log, cmd_options, row_list, term))
    else:
        # calibrate the map defined by the user
        calibrated_maps.append(calibrate_maps(log, command_options, row_list, term))

    return calibrated_maps
Esempio n. 4
0
def runPipeline():

    # the blessings.Terminal object is used for printing to the screen
    terminal = blessings.Terminal()
    term = terminal

    # create instance of CommandLine object to parse input, then
    # parse all the input parameters and store them as attributes in
    # cl_params structure
    cl = commandline.CommandLine()
    cl_params = cl.read(sys)

    # create a directory for storing log files
    mkdir_p('log')

    # instantiate a log object for log files and screen output
    log = Logging(cl_params, 'pipeline')

    # print a command summary to log file/terminal
    command_summary(cl_params, term, log)

    if not cl_params.imagingoff:

        # instantiate an Imaging object
        imag = Imaging.Imaging()

    else:
        imag = None

    log.doMessage(
        'INFO', '{t.underline}Start '
        'calibration.{t.normal}'.format(t=term))

    # calibrated_maps will contain a list of tuples each with a
    # MappingPipeline instance, window id, feed id and polarization id
    calibrated_maps = []

    # check to see if the infile parameter is a directory.  This is the
    # default case for vegas data processing.  If we have a directory of
    # files, we calibrate each input file separately, then perform the
    # imaging step on all the calibrated outputs.
    if os.path.isdir(cl_params.infilename):
        log.doMessage('INFO', 'Infile name is a directory')
        input_directory = cl_params.infilename

        # Instantiate a SdFits object for I/O and interpreting the
        #  contents of the index file
        sdf = SdFits()

        # generate a name for the index file based on the name of the
        #  raw SDFITS file.  The index file simply has a different extension
        directory_name = os.path.basename(cl_params.infilename.rstrip('/'))
        indexfile = cl_params.infilename + '/' + directory_name + '.index'
        try:
            # create a structure that lists the raw SDFITS rows for
            #  each scan/window/feed/polarization
            row_list, summary = sdf.parseSdfitsIndex(indexfile,
                                                     cl_params.mapscans)
        except IOError:
            log.doMessage('ERR', 'Could not open index file', indexfile)
            sys.exit()

        quitcal = False
        if cl_params.window and set(cl_params.window).isdisjoint(
                set(row_list.windows())):
            log.doMessage('ERR', 'no given WINDOW(S)', cl_params.window,
                          'in dataset')
            quitcal = True
        if cl_params.feed and set(cl_params.feed).isdisjoint(
                set(row_list.feeds())):
            log.doMessage('ERR', 'no given FEED(S)', cl_params.feed,
                          'in dataset')
            quitcal = True
        if cl_params.pol and set(cl_params.pol).isdisjoint(set(
                row_list.pols())):
            log.doMessage('ERR', 'no given POL(S)', cl_params.pol,
                          'in dataset')
            quitcal = True
        if cl_params.mapscans and set(cl_params.mapscans).isdisjoint(
                set(row_list.scans())):
            log.doMessage('ERR', 'no given MAPSCAN(S)', cl_params.mapscans,
                          'in dataset')
            quitcal = True
        if cl_params.refscans and set(cl_params.refscans).isdisjoint(
                set(row_list.scans())):
            log.doMessage('ERR', 'no given REFSCAN(S)', cl_params.refscans,
                          'in dataset')
            quitcal = True

        if quitcal:
            sys.exit(12)

        # calibrate one raw SDFITS file at a time
        for infilename in glob.glob(input_directory + '/' +
                                    os.path.basename(input_directory) +
                                    '*.fits'):
            log.doMessage('DBG', 'Attempting to calibrate',
                          os.path.basename(infilename).rstrip('.fits'))
            # change the infilename in the params structure to the
            #  current infile in the directory for each iteration
            cl_params.infilename = infilename

            # copy the cl_params structure so we can modify it during calibration
            # for each seperate file.
            commandline_options = copy.deepcopy(cl_params)
            calibrated_maps_this_file = calibrate_file(term, log,
                                                       commandline_options)
            if calibrated_maps_this_file:
                calibrated_maps.extend(calibrated_maps_this_file)
    else:
        commandline_options = copy.deepcopy(cl_params)
        calibrated_maps_this_file = calibrate_file(term, log,
                                                   commandline_options)
        if calibrated_maps_this_file:
            calibrated_maps.extend(calibrated_maps_this_file)

    if not calibrated_maps:
        log.doMessage('ERR',
                      'No calibrated spectra.  Check inputs and try again')
        sys.exit(-1)

    # if we are doing imaging
    if not cl_params.imagingoff:

        # image all the calibrated maps
        import itertools
        calibrated_maps = list(itertools.chain(*calibrated_maps))

        imag.run(log, term, cl_params, calibrated_maps)

    sys.stdout.write('\n')
Esempio n. 5
0
def calibrate_file(term, log, command_options):
    """Calibrate a single SDFITS file

       Actual calibration is done in calibrate_win_feed_pol(),
       which is called by calibrate_maps().

    """

    # Instantiate a SdFits object for I/O and interpreting the
    #  contents of the index file
    sdf = SdFits()

    # generate a name for the index file based on the name of the
    #  raw SDFITS file.  The index file simply has a different extension
    indexfile = sdf.nameIndexFile(command_options.infilename)
    try:
        # create a structure that lists the raw SDFITS rows for
        #  each scan/window/feed/polarization
        row_list, summary = sdf.parseSdfitsIndex(indexfile,
                                                 command_options.mapscans)
    except IOError:
        log.doMessage('ERR', 'Could not open index file', indexfile)
        sys.exit()

    log.doMessage('INFO', indexfile)
    log.doMessage('INFO', '    ', len(summary['WINDOWS']),
                  'spectral window(s)')
    for (win, freq) in sorted(summary['WINDOWS']):
        log.doMessage(
            'INFO', '       {win}: {freq:.4f} GHz'.format(win=win, freq=freq))
    log.doMessage('INFO', '    ', len(summary['FEEDS']), 'feed(s):',
                  ', '.join(sorted(summary['FEEDS'])))

    proceed_with_calibration = True

    # check for presence of pol(s) in dataset before attempting calibration
    if command_options.pol:
        if set(command_options.pol).isdisjoint(set(row_list.pols())):
            log.doMessage('WARN', 'POL', command_options.pol, 'not in',
                          os.path.basename(indexfile))
            proceed_with_calibration = False
        else:
            requested_pols = command_options.pol
            command_options.pol = list(
                set(command_options.pol).intersection(set(row_list.pols())))
            pol_diff = set(requested_pols).difference(set(command_options.pol))
            if pol_diff:
                log.doMessage('WARN', 'POL', list(pol_diff), 'not in',
                              os.path.basename(indexfile))

    # check for presence of feed(s) in dataset before attempting calibration
    if command_options.feed:
        if set(command_options.feed).isdisjoint(set(row_list.feeds())):
            log.doMessage('WARN', 'FEED', command_options.feed, 'not in',
                          os.path.basename(indexfile))
            proceed_with_calibration = False
        else:
            requested_feeds = command_options.feed
            command_options.feed = list(
                set(command_options.feed).intersection(set(row_list.feeds())))
            feed_diff = set(requested_feeds).difference(
                set(command_options.feed))
            if feed_diff:
                log.doMessage('WARN', 'FEED', list(feed_diff), 'not in',
                              os.path.basename(indexfile))

    # check for presence of window(s) in dataset before attempting calibration
    if command_options.window:
        if set(command_options.window).isdisjoint(set(row_list.windows())):
            log.doMessage('DBG', 'WINDOW', command_options.window, 'not in',
                          os.path.basename(indexfile))
            proceed_with_calibration = False
        else:
            requested_windows = command_options.window
            command_options.window = list(
                set(command_options.window).intersection(
                    set(row_list.windows())))
            window_diff = set(requested_windows).difference(
                set(command_options.window))
            if window_diff:
                log.doMessage('DBG', 'WINDOW', list(window_diff), 'not in',
                              os.path.basename(indexfile))

    if proceed_with_calibration is False:
        return None

    calibrated_maps = []
    # if there are no mapscans set at the command line, the user
    #  probably wants the pipeline to find maps in the input file
    if not command_options.mapscans:
        if command_options.refscans:
            log.doMessage(
                'WARN', 'Referene scan(s) given without map scans, '
                'ignoring reference scan settings.')
        command_options.refscans = []
        # this is where we try to guess the mapping blocks in the
        #  input file.  A powition-switched mapping block is defined
        #  as a reference scan, followed by mapping scans, optionally
        #  followed by another reference scan.  The returned structure
        #  is a list of tuples of (reference1, mapscans, reference2)
        maps = sdf.find_maps(indexfile)

        if maps:
            log.doMessage('INFO', 'Found', len(maps), 'map(s).')
        else:
            log.doMessage(
                'ERR', 'No scans specified and found no '
                'position-switched maps.')
            sys.exit()

        # calibrate each map found in the input file
        for map_number, map_params in enumerate(maps):
            cmd_options = set_map_scans(command_options, map_params)
            log.doMessage('INFO', '\nProcessing map:', str(map_number + 1),
                          'of', len(maps))
            calibrated_maps.append(
                calibrate_maps(log, cmd_options, row_list, term))
    else:
        # calibrate the map defined by the user
        calibrated_maps.append(
            calibrate_maps(log, command_options, row_list, term))

    return calibrated_maps