def main():
options = handle_args()
with open(options.listings) as f:
all_datasets, _ = driveami.load_listing(f,
expected_datatype=driveami.Datatype.ami_la_raw)
matching_datasets={}
for grp_name, grp_info in all_datasets.iteritems():
for fname in grp_info['files']:
if str.upper(options.match) in str.upper(str(fname)):
matching_datasets[grp_name]=grp_info
break
if len(matching_datasets)==0:
print("No matches found")
return 1
if options.outfile is None:
options.outfile = options.match+"_rawfiles.json"
print('Datasets matching "{}" written to'.format(options.match),
options.outfile)
with open(options.outfile, 'w') as f:
driveami.save_rawfile_listing(matching_datasets, f)
return 0
def test_rawfiles_roundtrip(self):
s = StringIO()
driveami.save_rawfile_listing(self.testdata, s)
listing, datatype = driveami.load_listing(StringIO(s.getvalue()))
self.assertEqual(datatype, driveami.Datatype.ami_la_raw)
self.assertEqual(listing, self.testdata)
def handle_args():
"""
Default values are defined here.
"""
parser = argparse.ArgumentParser(description="Calibrate raw AMI data and produce uvFITs")
parser.add_argument(
"-t",
"--topdir",
default=default_output_dir,
help="Top level data-output directory, default is : " + default_output_dir,
)
default_outfile = "calibrated_files.json"
parser.add_argument(
"-o",
"--outfile",
nargs="?",
default=default_outfile,
help="Specify filename for output listing of calibrated " "data, default: " + default_outfile,
)
parser.add_argument(
"groups_file",
metavar="groups_to_process.json",
nargs="?",
help="Specify file listing rawfiles for processing " "(overrides all other file options)",
)
parser.add_argument("--ami", default=default_ami_dir, help="Path to AMI directory, default: " + default_ami_dir)
parser.add_argument("-s", "--script", help="Specify non-standard reduction script")
parser.add_argument("-f", "--files", nargs="*", help="Specify individual files for reduction")
parser.add_argument(
"-g", "--group", dest="groupname", default="NOGROUP", help="Specify group name for individually specified files"
)
# parser.add_argument('-r', '--array', default='LA',
# help='Specify array (SA/LA) for individually specified files')
options = parser.parse_args()
options.ami_dir = os.path.expanduser(options.ami)
options.topdir = os.path.expanduser(options.topdir)
if options.script:
with open(options.script) as f:
options.script = f.read()
if options.groups_file:
print("Reducing files listed in:", options.groups_file)
with open(options.groups_file) as f:
data_groups, _ = driveami.load_listing(f, expected_datatype=driveami.Datatype.ami_la_raw)
elif options.files:
data_groups = {options.groupname: {"files": options.files}}
else:
parser.print_help()
sys.exit()
# print "SCRIPT:", options.script
return options, data_groups
def test_expected_cal(self):
s = StringIO()
driveami.save_rawfile_listing(self.testdata, s)
with self.assertRaises(ValueError):
listing, datatype = driveami.load_listing(StringIO(s.getvalue()),
expected_datatype=driveami.Datatype.ami_la_calibrated)
def test_check_for_magic_key(self):
s = StringIO()
json.dump(self.testdata, s)
with self.assertRaises(ValueError):
driveami.load_listing(s)
import os
# In[ ]:
def ami_utc_to_datetimes(utc_str):
"""Build 'datetime' objects from AMI-format date-strings"""
date_format = '%Y-%m-%d %H:%M:%S'
return datetime.datetime.strptime(utc_str, date_format)
# In[ ]:
data_dir = './'
with open(os.path.join(data_dir,'all_ami_rawfiles_metadata.json')) as f:
observations, _ = driveami.load_listing(f)
with open(os.path.join(data_dir,'all_ami_rawfiles_by_pointing.json')) as f:
groupings, _ = driveami.load_listing(f)
# In[ ]:
print("Loaded data for", len(observations), "observations, grouped into", len(groupings), "pointings")
# In[ ]:
#Drop rawtext - this is a big bunch of raw text data output from the AMI-REDUCE pipeline
#which is mainly just used for debugging - we're not interested in it here.
for k in observations.keys():
observations[k].pop(amikeys.raw_obs_text, None)
def handle_args():
"""
Default values are defined here.
"""
parser = argparse.ArgumentParser(
description="Calibrate raw AMI data and produce uvFITs")
parser.add_argument("-t", "--topdir", default=default_output_dir,
help="Top level data-output directory, default is : " +
default_output_dir)
default_outfile = "calibrated_files.json"
parser.add_argument('-o', '--outfile', nargs='?', default=default_outfile,
help='Specify filename for output listing of calibrated '
'data, default: ' + default_outfile)
parser.add_argument('groups_file', metavar='groups_to_process.json',
nargs='?',
help='Specify file listing rawfiles for processing '
'(overrides all other file options)')
parser.add_argument("--amidir", default=default_ami_dir,
help="Path to AMI directory, default: " + default_ami_dir)
parser.add_argument(
"--amiversion", default=default_ami_version,
help="AMI version (digital/legacy), default: " + default_ami_version,
choices=['digital','legacy'])
parser.add_argument('-s', '--script',
help='Specify non-standard reduction script')
parser.add_argument('-f', '--files', nargs='*',
help='Specify individual files for reduction')
parser.add_argument('-g', '--group', dest='groupname', default='NOGROUP',
help='Specify group name for individually specified files')
# parser.add_argument('-r', '--array', default='LA',
# help='Specify array (SA/LA) for individually specified files')
options = parser.parse_args()
options.amidir = os.path.expanduser(options.amidir)
options.topdir = os.path.expanduser(options.topdir)
if options.script:
with open(options.script) as f:
options.script = f.read()
if options.groups_file:
print("Reducing files listed in:", options.groups_file)
with open(options.groups_file) as f:
data_groups, _ = driveami.load_listing(f,
expected_datatype=driveami.Datatype.ami_la_raw)
elif options.files:
data_groups = {options.groupname: {'files': options.files}}
else:
parser.print_help()
sys.exit()
# print "SCRIPT:", options.script
return options, data_groups