def process_data_groups(data_groups, output_dir, ami_dir, array="LA", script=None): """Args: data_groups: Dictionary mapping groupname -> list of raw filenames output_dir: Folder where dataset group subfolders will be created. ami_dir: Top dir of the AMI ``reduce`` installation. array: 'LA' or 'SA' (Default: LA) """ if not script: script = driveami.scripts.standard_reduction processed_files_info = {} for grp_name in sorted(data_groups.keys()): try: r = driveami.Reduce(ami_dir, array=array) files = data_groups[grp_name][driveami.keys.files] grp_dir = os.path.join(output_dir, grp_name, "ami") driveami.ensure_dir(grp_dir) logger.info("Calibrating rawfiles and writing to {}".format(grp_dir)) for rawfile in files: try: logger.info("Reducing rawfile %s ...", rawfile) file_info = driveami.process_rawfile(rawfile, output_dir=grp_dir, reduce=r, script=script) except (ValueError, IOError) as e: logger.exception("Hit exception reducing file: %s\n" "Exception reads:\n%s\n", rawfile, e) continue # Also save the group assignment in the listings: file_info[driveami.keys.group_name] = grp_name processed_files_info[rawfile] = driveami.make_serializable(file_info) except Exception as e: logger.exception("Hit exception (probable timeout) reducing group: {}".format(grp_name)) continue return processed_files_info
def main(): options = handle_args() grouped_by_id_filename = options.outfile + '_by_id.json' grouped_by_pointing_filename = options.outfile + '_by_pointing.json' metadata_filename = options.outfile + '_metadata.json' r = driveami.Reduce(options.amidir, options.amiversion, options.array) logger.info("Loading observation metadata.") r.load_obs_info() logger.info("Grouping observations by target ID") id_groups = r.group_obs_by_target_id() with open(grouped_by_id_filename, 'w') as f: driveami.save_rawfile_listing(id_groups, f) logger.info("Grouping targets by pointing") pointing_groups = r.group_target_ids_by_pointing(id_groups, pointing_tolerance_in_degrees=0.5) with open(grouped_by_pointing_filename, 'w') as f: driveami.save_rawfile_listing(pointing_groups, f) with open(metadata_filename, 'w') as f: rawfile_dict = {fname: driveami.make_serializable(info) for fname, info in r.files.iteritems()} driveami.save_rawfile_listing(rawfile_dict, f) return 0
def test_list_files(self): self.assertEqual(len(self.reduce.files), 3) self.assertEqual(self.reduce.files.values()[0], {}) self.reduce.load_obs_info() self.assertNotEqual(self.reduce.files.values()[0], {}) s = StringIO.StringIO() rawfile_dict = {fname: driveami.make_serializable(info) for fname, info in self.reduce.files.iteritems()} driveami.save_rawfile_listing(rawfile_dict, s)
def main(): options = handle_args() grouped_by_id_filename = options.outfile + '_by_id.json' grouped_by_pointing_filename = options.outfile + '_by_pointing.json' metadata_filename = options.outfile + '_metadata.json' r = driveami.Reduce(options.amidir, options.amiversion, options.array) logger.info("Loading observation metadata.") r.load_obs_info() #Write file metadata with open(metadata_filename, 'w') as f: rawfile_dict = {fname: driveami.make_serializable(info) for fname, info in r.files.iteritems()} if not options.rawtext: for _, fdict in rawfile_dict.iteritems(): fdict.pop('raw_obs_listing_text') driveami.save_rawfile_listing(rawfile_dict, f) logger.info("Wrote file metadata listings to {}".format(metadata_filename)) #Write listings grouped by ID logger.info("Grouping observations by target ID") id_groups = r.group_obs_by_target_id() with open(grouped_by_id_filename, 'w') as f: driveami.save_rawfile_listing(id_groups, f) logger.info("Wrote id-grouped file-listings to {}".format(grouped_by_id_filename)) #Write listings grouped by pointing: logger.info("Grouping targets by pointing") pointing_groups = r.group_target_ids_by_pointing(id_groups, pointing_tolerance_in_degrees=0.5) with open(grouped_by_pointing_filename, 'w') as f: driveami.save_rawfile_listing(pointing_groups, f) logger.info( "Wrote pointing-grouped file-listings to {}".format( grouped_by_pointing_filename)) return 0