def main():
    options = handle_args()
    with open(options.listings) as f:
        all_datasets, _ = driveami.load_listing(f,
                                 expected_datatype=driveami.Datatype.ami_la_raw)

    matching_datasets={}
    for grp_name, grp_info in all_datasets.iteritems():
        for fname in grp_info['files']:
            if str.upper(options.match) in str.upper(str(fname)):
                matching_datasets[grp_name]=grp_info
                break

    if len(matching_datasets)==0:
        print("No matches found")
        return 1

    if options.outfile is None:
        options.outfile = options.match+"_rawfiles.json"
    print('Datasets matching "{}" written to'.format(options.match),
          options.outfile)
    with open(options.outfile, 'w') as f:
        driveami.save_rawfile_listing(matching_datasets, f)


    return 0
    def test_rawfiles_roundtrip(self):
        s = StringIO()
        driveami.save_rawfile_listing(self.testdata, s)

        listing, datatype = driveami.load_listing(StringIO(s.getvalue()))
        self.assertEqual(datatype, driveami.Datatype.ami_la_raw)
        self.assertEqual(listing, self.testdata)
def handle_args():
    """
    Default values are defined here.
    """
    parser = argparse.ArgumentParser(description="Calibrate raw AMI data and produce uvFITs")
    parser.add_argument(
        "-t",
        "--topdir",
        default=default_output_dir,
        help="Top level data-output directory, default is : " + default_output_dir,
    )

    default_outfile = "calibrated_files.json"

    parser.add_argument(
        "-o",
        "--outfile",
        nargs="?",
        default=default_outfile,
        help="Specify filename for output listing of calibrated " "data, default: " + default_outfile,
    )

    parser.add_argument(
        "groups_file",
        metavar="groups_to_process.json",
        nargs="?",
        help="Specify file listing rawfiles for processing " "(overrides all other file options)",
    )

    parser.add_argument("--ami", default=default_ami_dir, help="Path to AMI directory, default: " + default_ami_dir)

    parser.add_argument("-s", "--script", help="Specify non-standard reduction script")

    parser.add_argument("-f", "--files", nargs="*", help="Specify individual files for reduction")

    parser.add_argument(
        "-g", "--group", dest="groupname", default="NOGROUP", help="Specify group name for individually specified files"
    )

    # parser.add_argument('-r', '--array', default='LA',
    #                     help='Specify array (SA/LA) for individually specified files')

    options = parser.parse_args()
    options.ami_dir = os.path.expanduser(options.ami)
    options.topdir = os.path.expanduser(options.topdir)

    if options.script:
        with open(options.script) as f:
            options.script = f.read()

    if options.groups_file:
        print("Reducing files listed in:", options.groups_file)
        with open(options.groups_file) as f:
            data_groups, _ = driveami.load_listing(f, expected_datatype=driveami.Datatype.ami_la_raw)
    elif options.files:
        data_groups = {options.groupname: {"files": options.files}}
    else:
        parser.print_help()
        sys.exit()

    # print "SCRIPT:", options.script
    return options, data_groups
 def test_expected_cal(self):
     s = StringIO()
     driveami.save_rawfile_listing(self.testdata, s)
     with self.assertRaises(ValueError):
         listing, datatype = driveami.load_listing(StringIO(s.getvalue()),
                           expected_datatype=driveami.Datatype.ami_la_calibrated)
 def test_check_for_magic_key(self):
     s = StringIO()
     json.dump(self.testdata, s)
     with self.assertRaises(ValueError):
         driveami.load_listing(s)
Esempio n. 6
0
import os


# In[ ]:

def ami_utc_to_datetimes(utc_str):
    """Build 'datetime' objects from AMI-format date-strings"""
    date_format = '%Y-%m-%d %H:%M:%S'
    return datetime.datetime.strptime(utc_str, date_format)


# In[ ]:

data_dir = './'
with open(os.path.join(data_dir,'all_ami_rawfiles_metadata.json')) as f:
    observations, _ = driveami.load_listing(f)
with open(os.path.join(data_dir,'all_ami_rawfiles_by_pointing.json')) as f:
    groupings, _ = driveami.load_listing(f)


# In[ ]:

print("Loaded data for", len(observations), "observations, grouped into", len(groupings), "pointings")


# In[ ]:

#Drop rawtext - this is a big bunch of raw text data output from the AMI-REDUCE pipeline 
#which is mainly just used for debugging - we're not interested in it here.
for k in observations.keys():
    observations[k].pop(amikeys.raw_obs_text, None)
Esempio n. 7
0
def handle_args():
    """
    Default values are defined here.
    """
    parser = argparse.ArgumentParser(
        description="Calibrate raw AMI data and produce uvFITs")
    parser.add_argument("-t", "--topdir", default=default_output_dir,
                        help="Top level data-output directory, default is : " +
                             default_output_dir)

    default_outfile = "calibrated_files.json"

    parser.add_argument('-o', '--outfile', nargs='?', default=default_outfile,
                        help='Specify filename for output listing of calibrated '
                             'data, default: ' + default_outfile)

    parser.add_argument('groups_file', metavar='groups_to_process.json',
                        nargs='?',
                        help='Specify file listing rawfiles for processing '
                             '(overrides all other file options)')

    parser.add_argument("--amidir", default=default_ami_dir,
                        help="Path to AMI directory, default: " + default_ami_dir)

    parser.add_argument(
        "--amiversion", default=default_ami_version,
        help="AMI version (digital/legacy), default: " + default_ami_version,
        choices=['digital','legacy'])

    parser.add_argument('-s', '--script',
                        help='Specify non-standard reduction script')

    parser.add_argument('-f', '--files', nargs='*',
                        help='Specify individual files for reduction')

    parser.add_argument('-g', '--group', dest='groupname', default='NOGROUP',
                        help='Specify group name for individually specified files')

    # parser.add_argument('-r', '--array', default='LA',
    #                     help='Specify array (SA/LA) for individually specified files')

    options = parser.parse_args()
    options.amidir = os.path.expanduser(options.amidir)
    options.topdir = os.path.expanduser(options.topdir)

    if options.script:
        with open(options.script) as f:
            options.script = f.read()

    if options.groups_file:
        print("Reducing files listed in:", options.groups_file)
        with open(options.groups_file) as f:
            data_groups, _ = driveami.load_listing(f,
                                                   expected_datatype=driveami.Datatype.ami_la_raw)
    elif options.files:
        data_groups = {options.groupname: {'files': options.files}}
    else:
        parser.print_help()
        sys.exit()

    # print "SCRIPT:", options.script
    return options, data_groups