def process_data_groups(data_groups, output_dir, ami_dir, array="LA", script=None):
"""Args:
data_groups: Dictionary mapping groupname -> list of raw filenames
output_dir: Folder where dataset group subfolders will be created.
ami_dir: Top dir of the AMI ``reduce`` installation.
array: 'LA' or 'SA' (Default: LA)
"""
if not script:
script = driveami.scripts.standard_reduction
processed_files_info = {}
for grp_name in sorted(data_groups.keys()):
try:
r = driveami.Reduce(ami_dir, array=array)
files = data_groups[grp_name][driveami.keys.files]
grp_dir = os.path.join(output_dir, grp_name, "ami")
driveami.ensure_dir(grp_dir)
logger.info("Calibrating rawfiles and writing to {}".format(grp_dir))
for rawfile in files:
try:
logger.info("Reducing rawfile %s ...", rawfile)
file_info = driveami.process_rawfile(rawfile, output_dir=grp_dir, reduce=r, script=script)
except (ValueError, IOError) as e:
logger.exception("Hit exception reducing file: %s\n" "Exception reads:\n%s\n", rawfile, e)
continue
# Also save the group assignment in the listings:
file_info[driveami.keys.group_name] = grp_name
processed_files_info[rawfile] = driveami.make_serializable(file_info)
except Exception as e:
logger.exception("Hit exception (probable timeout) reducing group: {}".format(grp_name))
continue
return processed_files_info
def test_reduction(self):
file_info = driveami.process_rawfile(self.rawfile_basename,
output_dir=resources.driveami_testdir,
reduce=self.reduce,
script=standard_legacy_reduction)
output_fits = file_info[keys.target_uvfits]
self.assertTrue(os.path.isfile(output_fits))
def ami_rawfile_quicklook(filename, ami_dir, casa_dir, output_dir):
"""A data reduction subroutine (specific to user's application)."""
rawfile = os.path.basename(filename)
groupname = rawfile.split('-')[0]
group_dir = os.path.join(output_dir, groupname)
group_ami_outdir = os.path.join(group_dir, 'ami')
group_casa_outdir = os.path.join(group_dir, 'casa')
group_fits_outdir = os.path.join(group_dir, 'images')
try:
with driveami.Reduce(ami_dir,
ami_version=driveami.AmiVersion.digital) as reduce:
obs_info = driveami.process_rawfile(
rawfile, group_ami_outdir, reduce,
script=driveami.scripts.standard_digital_reduction)
image_casa_outdir = os.path.join(group_casa_outdir,
obs_info[amikeys.obs_name])
casa_logfile = os.path.join(group_casa_outdir, 'casalog.txt')
casa_script = []
vis = drivecasa.commands.import_uvfits(casa_script,
uvfits_path=obs_info[amikeys.target_uvfits],
out_dir=image_casa_outdir,
overwrite=True)
dirty_maps = drivecasa.commands.clean(casa_script,
vis_paths=vis,
niter=200,
threshold_in_jy=2.5 * obs_info[amikeys.est_noise_jy],
mask='',
other_clean_args=ami_clean_args,
out_dir=image_casa_outdir,
overwrite=True)
dirty_maps_fits_image = drivecasa.commands.export_fits(casa_script,
image_path=dirty_maps.image,
out_dir=group_fits_outdir,
overwrite=True)
casa = drivecasa.Casapy(casa_logfile=casa_logfile,
casa_dir=casa_dir)
casa.run_script(casa_script, raise_on_severe=True)
except Exception as e:
error_message = ("Hit exception reducing file: %s, exception reads:\n%s\n"
% (rawfile, e))
logger.exception(e)
return error_message
return "Successfully processed " + filename
