def set_centroids(self, source, slot, alg=8, apply_dt=True): """ Assign centroids from ``source`` and ``slot`` to the objects centroid attributes (yag, zag, yag_times, zag_times) For the supported sources (ground, obc) the centroids are fetched from the mica L1 archive or telemetry. yag, zag, yag_times an zag_times can also be set directly without use of this method. :param source: 'ground' | 'obc' :param slot: ACA slot :param alg: for ground processing, use centroids from this algorithm. :param apply_dt: apply centroid time offsets via 'set_offsets' """ self.centroid_source = source self.centroid_dt = None start = self.start stop = self.stop # Get centroids from Ska eng archive or mica L1 archive if source == 'ground': acen_files = sorted( asp_l1.get_files(start=start, stop=stop, content=['ACACENT'])) acen = vstack([Table.read(f) for f in sorted(acen_files)], metadata_conflicts='silent') ok = ((acen['slot'] == slot) & (acen['alg'] == alg) & (acen['status'] == 0) & (acen['time'] >= DateTime(start).secs) & (acen['time'] <= DateTime(stop).secs)) yags = np.array(acen[ok]['ang_y'] * 3600) zags = np.array(acen[ok]['ang_z'] * 3600) yag_times = np.array(acen[ok]['time']) zag_times = np.array(acen[ok]['time']) elif source == 'obc': telem = fetch.Msidset( ['AOACYAN{}'.format(slot), 'AOACZAN{}'.format(slot)], start, stop) # Filter centroids for reasonble-ness yok = telem['AOACYAN{}'.format(slot)].vals > -3276 zok = telem['AOACZAN{}'.format(slot)].vals > -3276 yags = telem['AOACYAN{}'.format(slot)].vals[yok] yag_times = telem['AOACYAN{}'.format(slot)].times[yok] zags = telem['AOACZAN{}'.format(slot)].vals[zok] zag_times = telem['AOACZAN{}'.format(slot)].times[zok] else: raise ValueError("centroid_source must be 'obc' or 'ground'") self.yags = yags self.yag_times = yag_times self.zags = zags self.zag_times = zag_times if apply_dt is True: self.set_offsets() else: self.centroid_dt = 0
def set_centroids(self, source, slot, alg=8, apply_dt=True): """ Assign centroids from ``source`` and ``slot`` to the objects centroid attributes (yag, zag, yag_times, zag_times) For the supported sources (ground, obc) the centroids are fetched from the mica L1 archive or telemetry. yag, zag, yag_times an zag_times can also be set directly without use of this method. :param source: 'ground' | 'obc' :param slot: ACA slot :param alg: for ground processing, use centroids from this algorithm. :param apply_dt: apply centroid time offsets via 'set_offsets' """ self.centroid_source = source self.centroid_dt = None start = self.start stop = self.stop # Get centroids from Ska eng archive or mica L1 archive if source == 'ground': acen_files = sorted(asp_l1.get_files(start=start, stop=stop, content=['ACACENT'])) acen = vstack([Table.read(f) for f in sorted(acen_files)], metadata_conflicts='silent') ok = ((acen['slot'] == slot) & (acen['alg'] == alg) & (acen['status'] == 0) & (acen['time'] >= DateTime(start).secs) & (acen['time'] <= DateTime(stop).secs)) yags = np.array(acen[ok]['ang_y'] * 3600) zags = np.array(acen[ok]['ang_z'] * 3600) yag_times = np.array(acen[ok]['time']) zag_times = np.array(acen[ok]['time']) elif source == 'obc': telem = fetch.Msidset(['AOACYAN{}'.format(slot), 'AOACZAN{}'.format(slot)], start, stop) # Filter centroids for reasonble-ness yok = telem['AOACYAN{}'.format(slot)].vals > -3276 zok = telem['AOACZAN{}'.format(slot)].vals > -3276 yags = telem['AOACYAN{}'.format(slot)].vals[yok] yag_times = telem['AOACYAN{}'.format(slot)].times[yok] zags = telem['AOACZAN{}'.format(slot)].vals[zok] zag_times = telem['AOACZAN{}'.format(slot)].times[zok] else: raise ValueError("centroid_source must be 'obc' or 'ground'") self.yags = yags self.yag_times = yag_times self.zags = zags self.zag_times = zag_times if apply_dt is True: self.set_offsets() else: self.centroid_dt = 0
def test_get_l0_images(): """ Do a validation test of get_l0_images: - Get 20 mins of image data for slot 6 of obsid 8008 (very nice clean stars) - Do first moment centroids in row and col - Compare to aspect pipeline FM centroids for same slot data This is a deep test that all the signs are right. If not then everything breaks badly because the star image doesn't move in sync with row0, col0. """ start = '2007:002:06:00:00' stop = '2007:002:06:20:00' imgs = aca_l0.get_l0_images(start, stop, slot=6) files = asp_l1.get_files(8008, content=['ACACENT']) acen = Table.read(files[0]) # Pick FM centroids for slot 6 ok = (acen['alg'] == 1) & (acen['slot'] == 6) acen = acen[ok] # Row and col centroids rcs = [] ccs = [] times = [img.TIME for img in imgs] # Easy way to do FM centroids with mgrid rw, cw = np.mgrid[0:6, 0:6] # rw = [[0, 0, 0, 0, 0, 0], # [1, 1, 1, 1, 1, 1], # [2, 2, 2, 2, 2, 2], # [3, 3, 3, 3, 3, 3], # [4, 4, 4, 4, 4, 4], # [5, 5, 5, 5, 5, 5]] for img in imgs: norm = np.sum(img) rcs.append(np.sum(img * rw) / norm + img.row0) ccs.append(np.sum(img * cw) / norm + img.col0) rcen = interpolate(acen['cent_i'], acen['time'], times) ccen = interpolate(acen['cent_j'], acen['time'], times) assert np.all(np.abs(rcen - rcs) < 0.05) assert np.all(np.abs(ccen - ccs) < 0.05)
def test_get_slot_data_8x8(): """ Do a validation test of get_l0_images: - Get 20 mins of image data for slot 6 of obsid 8008 (very nice clean stars) - Do first moment centroids in row and col - Compare to aspect pipeline FM centroids for same slot data This is a deep test that all the signs are right. If not then everything breaks badly because the star image doesn't move in sync with row0, col0. """ start = '2007:002:06:00:00' stop = '2007:002:06:20:00' slot_data = aca_l0.get_slot_data(start, stop, slot=6, centered_8x8=True) files = asp_l1.get_files(8008, content=['ACACENT']) acen = Table.read(files[0]) # Pick FM centroids for slot 6 ok = (acen['alg'] == 1) & (acen['slot'] == 6) acen = acen[ok] # Row and col centroids times = slot_data['TIME'] # Easy way to do FM centroids with mgrid rw, cw = np.mgrid[0:8, 0:8] img_raw = slot_data['IMGRAW'] # np.round(slot_data['IMGRAW']).astype(int) norm = np.sum(img_raw, axis=(1, 2)) rcs = np.sum(img_raw * rw, axis=(1, 2)) / norm + slot_data['IMGROW0'] - 1 ccs = np.sum(img_raw * cw, axis=(1, 2)) / norm + slot_data['IMGCOL0'] - 1 rcen = interpolate(acen['cent_i'], acen['time'], times) ccen = interpolate(acen['cent_j'], acen['time'], times) assert np.all(np.abs(rcen - rcs) < 0.05) assert np.all(np.abs(ccen - ccs) < 0.05)
def update(obsids, config=None): from mica.archive import asp_l1 """ For a list of obsids, update the mica table of aspect 1 processing """ if config is None: config = DEFAULT_CONFIG logger = logging.getLogger('asp 1 proc table') logger.setLevel(logging.INFO) if not len(logger.handlers): logger.addHandler(logging.StreamHandler()) apstat_db = dict(dbi='sybase', server='sqlsao', database='axafapstat') proc_db_file = os.path.join(MICA_ARCHIVE, 'asp1', 'processing_asp_l1.db3') if not os.path.exists(proc_db_file) or os.stat(proc_db_file).st_size == 0: if not os.path.exists(config['data_root']): os.makedirs(config['data_root']) logger.info("creating aspect_1_proc db from {}".format( config['sql_def'])) db_sql = Path(__file__).parent / config['sql_def'] db_init_cmds = open(db_sql).read() proc_db = Ska.DBI.DBI(dbi='sqlite', server=proc_db_file) proc_db.execute(db_init_cmds) else: proc_db = Ska.DBI.DBI(dbi='sqlite', server=proc_db_file) archdb = Ska.DBI.DBI(dbi='sqlite', server=os.path.join(config['data_root'], 'archfiles.db3')) for obs in obsids: logger.info("Adding asp1 processing for obs {}".format(obs)) asols = asp_l1.get_files(obsid=obs, content='ASPSOL', revision='all') for sol in asols: logger.info("\tprocessing {}".format(sol)) procdir = os.path.dirname(sol) # As of DS 10.8.3, there are both "com" logs and per-ai logs. # This glob should get the per-ai logs. We use the first one # to get an obspar version. logfiles = get_globfiles(os.path.join(procdir, "asp_l1_f*log*"), minfiles=1, maxfiles=None) aspect_log = gzip.open(logfiles[0], 'rt').read() # read the obspar version with a regex from the log obspar_version = int( re.search(r"axaff\d{5}_\d{3}N(\d{3})_obs0a\.par", aspect_log).group(1)) hdus = fits.open(sol) obi = hdus[1].header['OBI_NUM'] revision = hdus[1].header['REVISION'] with Ska.DBI.DBI(**apstat_db) as db: aspect_1 = db.fetchall("""SELECT * FROM aspect_1 WHERE obsid = {obsid} AND obi = {obi} AND revision = {revision} """.format(obsid=obs, obi=obi, revision=revision)) if len(aspect_1) > 1: raise ValueError indb = proc_db.fetchall("""SELECT * from aspect_1_proc WHERE aspect_1_id = {} """.format(aspect_1[0]['aspect_1_id'])) archrec = archdb.fetchall("""select * from archfiles where obsid = {obsid} and revision = {revision} and content = 'ASPSOL' """.format(obsid=obs, revision=revision)) # if already in the database, reset the V&V # it doesn't really need to be redone (could just be relinked) # but this should at least make the bookkeeping consistent if len(indb): logger.info("Resetting vv_complete to 0 for obsid {obsid} rev {revision}".format( obsid=obs, revision=revision)) proc_db.execute("""UPDATE aspect_1_proc SET vv_complete = 0 WHERE obsid = {obsid} AND revision = {revision} """.format(obsid=obs, revision=revision)) else: proc_db.insert(dict(aspect_1_id=aspect_1[0]['aspect_1_id'], obsid=obs, obi=obi, revision=revision, obspar_version=obspar_version, ap_date=str(aspect_1[0]['ap_date'])), 'aspect_1_proc') isdefault = 'NULL' if archrec[0]['isdefault'] is None else 1 proc_db.execute(f"""UPDATE aspect_1_proc SET isdefault = {isdefault} WHERE obsid = {obs} AND revision = {revision} """) logger.info("\tUpdated table for {}".format(obs))
def update(obsids, config=None): from mica.archive import asp_l1 """ For a list of obsids, update the mica table of aspect 1 processing """ if config is None: config = DEFAULT_CONFIG logger = logging.getLogger('asp 1 proc table') logger.setLevel(logging.INFO) if not len(logger.handlers): logger.addHandler(logging.StreamHandler()) apstat_db = dict(dbi='sybase', server='sqlsao', database='axafapstat') proc_db_file = os.path.join(MICA_ARCHIVE, 'asp1', 'processing_asp_l1.db3') if not os.path.exists(proc_db_file) or os.stat(proc_db_file).st_size == 0: if not os.path.exists(config['data_root']): os.makedirs(config['data_root']) logger.info("creating aspect_1_proc db from {}".format( config['sql_def'])) db_sql = os.path.join(os.environ['SKA_DATA'], 'mica', config['sql_def']) db_init_cmds = file(db_sql).read() proc_db = dict(dbi='sqlite', server=proc_db_file) proc_db.execute(db_init_cmds) else: proc_db = Ska.DBI.DBI(dbi='sqlite', server=proc_db_file) archdb = Ska.DBI.DBI(dbi='sqlite', server=os.path.join(config['data_root'], 'archfiles.db3')) for obs in obsids: logger.info("Adding asp1 processing for obs {}".format(obs)) asols = asp_l1.get_files(obsid=obs, content='ASPSOL', revision='all') for sol in asols: logger.info("\tprocessing {}".format(sol)) procdir = os.path.dirname(sol) logfile = glob(os.path.join(procdir, "*log*"))[0] aspect_log = gzip.open(logfile).read() # read the obspar version with a regex from the log obspar_version = int( re.search("axaff\d{5}_\d{3}N(\d{3})_obs0a\.par", aspect_log).group(1)) hdus = fits.open(sol) obi = hdus[1].header['OBI_NUM'] revision = hdus[1].header['REVISION'] with Ska.DBI.DBI(**apstat_db) as db: aspect_1 = db.fetchall("""SELECT * FROM aspect_1 WHERE obsid = {obsid} AND obi = {obi} AND revision = {revision} """.format(obsid=obs, obi=obi, revision=revision)) if len(aspect_1) > 1: raise ValueError indb = proc_db.fetchall("""SELECT * from aspect_1_proc WHERE aspect_1_id = {} """.format(aspect_1[0]['aspect_1_id'])) archrec = archdb.fetchall("""select * from archfiles where obsid = {obsid} and revision = {revision} and content = 'ASPSOL' """.format(obsid=obs, revision=revision)) # if already in the database, reset the V&V # it doesn't really need to be redone (could just be relinked) # but this should at least make the bookkeeping consistent if len(indb): logger.info("Resetting vv_complete to 0 for obsid {obsid} rev {revision}".format( obsid=obs, revision=revision)) proc_db.execute("""UPDATE aspect_1_proc SET vv_complete = 0 WHERE obsid = {obsid} AND revision = {revision} """.format(obsid=obs, revision=revision)) else: proc_db.insert(dict(aspect_1_id=aspect_1[0]['aspect_1_id'], obsid=obs, obi=obi, revision=revision, obspar_version=obspar_version, ap_date=str(aspect_1[0]['ap_date'])), 'aspect_1_proc') isdefault = archrec[0]['isdefault'] if isdefault is not None: proc_db.execute("""UPDATE aspect_1_proc SET isdefault = {isdefault} WHERE obsid = {obsid} AND revision = {revision} """.format(isdefault=archrec[0]['isdefault'], obsid=obs, revision=revision)) logger.info("\tUpdated table for {}".format(obs))
idx = np.unique(obs['obsid'], return_index=True)[1] obs = Table(obs[idx]) obs.sort('tstart') obs['datestart'] = Time(obs['tstart'], format='cxcsec').yday obs.pprint(max_lines=-1) obsid_index = shelve.open(obsid_file) # Go through obsids and either process or skip for obsid in obs['obsid']: if str(obsid) in obsid_index: logger.info('Skipping obsid {} - already in archive'.format(obsid)) continue logger.info('Processing obsid {}'.format(obsid)) asol_files = sorted(asp_l1.get_files(obsid=obsid, content='ASPSOL')) if not asol_files: logger.info('Skipping obsid {} - no asol files'.format(obsid)) continue asol = get_asol(obsid, asol_files, opt.dt) add_asol_to_h5(h5_file, asol) obsid_index[str(obsid)] = asol_files obsid_index.close() logger.info('File {} updated'.format(h5_file)) logger.info('File {} updated'.format(obsid_file)) # Write out to FITS fits_file = re.sub(r'\.h5$', '.fits', h5_file)