def get_state_data(tstart, tstop): """ Get states where 'vid_board', 'clocking', 'fep_count', 'pcad_mode' are constant. Args: tstart (int, float, string): Start time, using Chandra.Time epoch tstop (int, float, string): Stop time, using Chandra.Time epoch Returns: (numpy.ndarray): state data """ keys = [ 'pitch', 'off_nom_roll', 'ccd_count', 'fep_count', 'clocking', 'vid_board', 'pcad_mode', 'simpos' ] state_data = states.get_states(tstart, tstop, state_keys=keys, merge_identical=True) # Convert 'trans_keys' elements from 'TransKeysSet' objects to strings for compatibility with the 'reduce_states' # function in the 'Chandra.cmd_states' package state_data['trans_keys'] = [str(val) for val in state_data['trans_keys']] state_data['tstart'] = DateTime( state_data['datestart']).secs # Add start time in seconds as 'tstart' state_data['tstop'] = DateTime( state_data['datestop']).secs # Add stop time in seconds as 'tstop' # relying on 'pcad_mode' to ensure attitude does not change significantly within a dwell state_data = reduce_states( state_data, ['ccd_count', 'fep_count', 'clocking', 'vid_board', 'pcad_mode']) return state_data
def del_stats(colname, time0, interval): """Delete all rows in ``interval`` stats file for column ``colname`` that occur after time ``time0`` - ``interval``. This is used to fix problems that result from a file misorder. Subsequent runs of update_stats will refresh the values correctly. """ dt = {'5min': 328, 'daily': 86400}[interval] ft['msid'] = colname ft['interval'] = interval stats_file = msid_files['stats'].abs if not os.path.exists(stats_file): raise IOError('Stats file {} not found'.format(stats_file)) logger.info('Fixing stats file %s after time %s', stats_file, DateTime(time0).date) stats = tables.openFile(stats_file, mode='a', filters=tables.Filters(complevel=5, complib='zlib')) index0 = time0 // dt - 1 indexes = stats.root.data.col('index')[:] row0 = np.searchsorted(indexes, [index0])[0] - 1 if opt.dry_run: n_del = len(stats.root.data) - row0 else: n_del = stats.root.data.removeRows(row0, len(stats.root.data)) logger.info('Deleted %d rows from row %s (%s) to end', n_del, row0, DateTime(indexes[row0] * dt).date) stats.close()
def test_start_day(self): self.assertEqual( DateTime('1996365.010203').day_start().iso, '1996-12-30 00:00:00.000') self.assertEqual( DateTime('1996367.010203').day_start().iso, '1997-01-01 00:00:00.000')
def test_sun_vec_versus_telemetry(): """ Test sun vector values `pitch` and `off_nominal_roll` versus flight telem. Include Load maneuver at 2017:349:20:52:37.719 in DEC1117 with large pitch and off_nominal_roll change (from around zero to -17 deg). State values are within 1.5 degrees of telemetry. """ state_keys = ['pitch', 'off_nom_roll'] start, stop = '2017:349:10:00:00', '2017:350:10:00:00' cmds = commands.get_cmds(start, stop) rk = states.get_states(state_keys=state_keys, cmds=cmds, merge_identical=True)[-20:-1] tstart = DateTime(rk['datestart']).secs tstop = DateTime(rk['datestop']).secs tmid = (tstart + tstop) / 2 # Pitch from telemetry dat = fetch.Msid('pitch', tstart[0] - 100, tstop[-1] + 100) dat.interpolate(times=tmid) delta = np.abs(dat.vals - rk['pitch']) assert np.max(rk['pitch']) - np.min(rk['pitch']) > 75 # Big maneuver assert np.all(delta < 1.5) # Off nominal roll (not roll from ra,dec,roll) from telemetry dat = fetch.Msid('roll', tstart[0] - 100, tstop[-1] + 100) dat.interpolate(times=tmid) delta = np.abs(dat.vals - rk['off_nom_roll']) assert np.max(rk['off_nom_roll']) - np.min( rk['off_nom_roll']) > 20 # Large range assert np.all(delta < 1.5)
def _fetch_by_time(self, range_tstart, range_tstop): logger.info("Fetching %s from %s to %s" % ('ACA L0 Data', DateTime(range_tstart).date, DateTime(range_tstop).date)) archfiles = self._get_archive_files(DateTime(range_tstart), DateTime(range_tstop)) return archfiles
def test_secs(self): self.assertEqual('%.3f' % DateTime(20483020.).secs, '20483020.000') self.assertEqual(DateTime(20483020.).date, '1998:238:01:42:36.816') self.assertEqual( DateTime('2012:001:00:00:00.000').secs, 441763266.18399996) self.assertEqual( DateTime(473385667.18399996).date, '2013:001:00:00:00.000')
def get_ifot(event_type, start=None, stop=None, props=[], columns=[], timeout=TIMEOUT, types={}): start = DateTime('1998:001' if start is None else start) stop = DateTime(stop) event_props = '.'.join([event_type] + props) params = odict(r='home', t='qserver', format='tsv', tstart=start.date, tstop=stop.date, e=event_props, ul='7', ) if columns: params['columns'] = ','.join(columns) # Get the TSV data for the iFOT event table url = ROOTURL + URLS['ifot'] response = requests.get(url, auth=get_auth(), params=params, timeout=timeout) # For Py2 convert from unicode to ASCII str text = response.text text = re.sub(r'\r\n', ' ', text) lines = [x for x in text.split('\t\n') if x.strip()] converters = {key: [ascii.convert_numpy(getattr(np, type_))] for key, type_ in types.items()} dat = ascii.read(lines, format='tab', guess=False, converters=converters, fill_values=None) return dat
def main(): opt = get_opt() pitchs = list(range(45, 170, 5)) model_spec = json.load(open(opt.model_spec, 'r')) plt.close(1) plt.figure(1, figsize=(6, 4)) years = np.arange(opt.start_year, opt.stop_year + 0.0001, opt.dt_year) times = DateTime(years, format='frac_year').secs for tstart in times: pitch_temps = [] for pitch in pitchs: model = calc_model(tstart, pitch, model_spec) pitch_temps.append(model.comp['pftank2t'].mvals[-1]) pitch_temps = np.array(pitch_temps) * 1.8 + 32 # In degF label = '{}'.format(DateTime(tstart).date[:8]) plt.plot(pitchs, pitch_temps, lw=1.5, label=label) plt.legend(loc='best') plt.grid('on') plt.ylabel('PFTANK2T (degF)') plt.title('Tank settling temperature') plt.tight_layout() plt.legend(loc='best') if opt.plot_file: plt.savefig(opt.plot_file)
def get_unique_orbit_cmds(orbit_cmds): """ Given list of ``orbit_cmds`` find the quasi-unique set. In the event of a replan/reopen or other schedule oddity, it can happen that there are multiple cmds that describe the same orbit event. Since the detailed timing might change between schedule runs, cmds are considered the same if the date is within 3 minutes. """ if len(orbit_cmds) == 0: return [] # Sort by (event_type, date) orbit_cmds.sort(key=lambda y: (y['params']['EVENT_TYPE'], y['date'])) uniq_cmds = [orbit_cmds[0]] # Step through one at a time and add to uniq_cmds only if the candidate is # "different" from uniq_cmds[-1]. for cmd in orbit_cmds: last_cmd = uniq_cmds[-1] if (cmd['params']['EVENT_TYPE'] == last_cmd['params']['EVENT_TYPE'] and abs(DateTime(cmd['date']).secs - DateTime(last_cmd['date']).secs) < 180): # Same event as last (even if date is a bit different). Now if this one # has a larger timeline_id that means it is from a more recent schedule, so # use that one. if cmd['timeline_id'] > last_cmd['timeline_id']: uniq_cmds[-1] = cmd else: uniq_cmds.append(cmd) uniq_cmds.sort(key=lambda y: y['date']) return uniq_cmds
def test_init_from_mxDateTime(self): if HAS_MX_DATETIME: mxd = DateTime('1999-01-01 12:13:14').mxDateTime self.assertEqual(DateTime(mxd).fits, '1999-01-01T12:14:18.184') self.assertEqual( DateTime(mxd).mxDateTime.strftime('%c'), 'Fri Jan 1 12:13:14 1999')
def get_eclipse_text(eclfile, t1, t2): #---------------------------------- # Import eclipse data eclipse = ecl.read_eclipse_file(eclfile) # eclipse = ecl.read_eclipse_file('./ECLIPSE_HISTORY_2014.txt') #---------------------------------- # Look for eclipses that fall in the current week ecllist = [] for k in eclipse['eclipse_nums']: tstart = eclipse[k]['entrancepenumbra']['startsec'] if eclipse[k].has_key('exitpenumbra'): # This is used for a normal eclipse tstop = eclipse[k]['exitpenumbra']['stopsec'] else: # This is used for a penumbral eclipse tstop = eclipse[k]['entrancepenumbra']['stopsec'] if (tstart >= DateTime(t1).secs) & (tstop <= DateTime(t2).secs): ecllist.append(k) if ecllist: eclipselines = writeEclipseText(eclipse, ecllist) else: eclipselines = 'None.' return eclipselines
def get_bad_mask(tlm): mask = np.zeros(len(tlm), dtype='bool') for interval in characteristics.bad_times: bad = ((tlm['date'] >= DateTime(interval['start']).secs) & (tlm['date'] < DateTime(interval['stop']).secs)) mask[bad] = True return mask
def fetch(self, msid, attr='vals', method='linear'): """Get data from the Chandra engineering archive. Parameters ---------- msid : attr : (Default value = 'vals') method : (Default value = 'linear') Returns ------- """ tpad = DEFAULT_DT * 5.0 datestart = DateTime(self.tstart - tpad).date datestop = DateTime(self.tstop + tpad).date logger.info('Fetching msid: %s over %s to %s' % (msid, datestart, datestop)) try: import Ska.engarchive.fetch_sci as fetch tlm = fetch.MSID(msid, datestart, datestop, stat='5min') tlm.filter_bad_times() except ImportError: raise ValueError('Ska.engarchive.fetch not available') if tlm.times[0] > self.tstart or tlm.times[-1] < self.tstop: raise ValueError('Fetched telemetry does not span model start and ' 'stop times for {}'.format(msid)) vals = Ska.Numpy.interpolate(getattr(tlm, attr), tlm.times, self.times, method=method) return vals
def psf_time_anal(sources, title): """ Time analysis - refit only the current year, somehow remember or read from file coefficients for the previous years? They will change if we change the source detection algorithm? """ if self.year is None: current_year = DateTime().year # years = np.arange(1999, current_year + 1) years = [current_year] for year in years: self.year = year flt1 = DateTime(sources['TSTART']).date >= np.str(year) flt2 = DateTime(sources['TSTART']).date < np.str(year + 1) flt_sources = sources[flt1 & flt2] if len(flt_sources) > NUM_FIT_SRC: flt_sources.sort('DIST_ARCSEC') x = flt_sources['DIST_ARCSEC'] y = flt_sources['PSF_ARCSEC'] self.popt, self.pcov = curve_fit(psf_function, x, y) popt, pcov = curve_fit(psf_function_a0, x, y) self.on_axis = popt[0] self.on_axis_error = pcov[0][0] self.x = x self.y = y
def get_states(datestart, datestop): """Get states exactly covering date range :param datestart: start date :param datestop: stop date :param db: database handle :returns: np recarry of states """ get_states_func = get_states_kadi if opt.dbi == 'kadi' else get_states_dbi states = get_states_func(datestart, datestop) # Add power columns to states and tlm states = Ska.Numpy.add_column(states, 'power', get_power(states)) # Set start and end state date/times to match telemetry span. Extend the # state durations by a small amount because of a precision issue converting # to date and back to secs. (The reference tstop could be just over the # 0.001 precision of date and thus cause an out-of-bounds error when # interpolating state values). states[0].tstart = DateTime(datestart).secs - 0.01 states[0].datestart = DateTime(states[0].tstart).date states[-1].tstop = DateTime(datestop).secs + 0.01 states[-1].datestop = DateTime(states[-1].tstop).date return states
def interpolate_orbit_points(orbit_points, name): """ Linearly interpolate across any gaps for ``name`` orbit_points. """ if len(orbit_points) == 0: return [] ok = orbit_points['name'] == name ops = orbit_points[ok] # Get the indexes of missing orbits idxs = np.flatnonzero(np.diff(ops['orbit_num']) > 1) new_orbit_points = [] for idx in idxs: op0 = ops[idx] op1 = ops[idx + 1] orb_num0 = op0['orbit_num'] orb_num1 = op1['orbit_num'] time0 = DateTime(op0['date']).secs time1 = DateTime(op1['date']).secs for orb_num in range(orb_num0 + 1, orb_num1): time = time0 + (orb_num - orb_num0) / (orb_num1 - orb_num0) * (time1 - time0) date = DateTime(time).date new_orbit_point = (date, name, orb_num, op0['descr']) logger.info('Adding new orbit point {}'.format(new_orbit_point)) new_orbit_points.append(new_orbit_point) return new_orbit_points
def get_telem_values(tstart, msids, days=14, dt=32.8, name_map={}): """ Fetch last ``days`` of available ``msids`` telemetry values before time ``tstart``. :param tstart: start time for telemetry (secs) :param msids: fetch msids list :param days: length of telemetry request before ``tstart`` :param dt: sample time (secs) :param name_map: dict mapping msid to recarray col name :returns: np recarray of requested telemetry values from fetch """ tstart = DateTime(tstart).secs start = DateTime(tstart - days * 86400).date stop = DateTime(tstart).date logger.info('Fetching telemetry between %s and %s' % (start, stop)) msidset = fetch.Msidset(msids, start, stop) start = max(x.times[0] for x in msidset.values()) stop = min(x.times[-1] for x in msidset.values()) msidset.interpolate(dt, start, stop) # Finished when we found at least 10 good records (5 mins) if len(msidset.times) < 10: raise ValueError('Found no telemetry within %d days of %s' % (days, str(tstart))) outnames = ['date'] + [name_map.get(x, x) for x in msids] out = np.rec.fromarrays([msidset.times] + [msidset[x].vals for x in msids], names=outnames) return out
def download_data(start=None, stop=None): """ Extract data to compute HRMA focus plots. :param start, stop: any DateTime compatible time formats :returns: fetches acis*evt2.fits.gz, hrc*evt2.fits.gz files from the archive and stores them in the current directory """ # Remove directory 'param' and all files that it contains if os.path.isdir('param'): rmtree('param') os.makedirs('param') # check whether previous fits files are still around # if yes, remove them fits_files = glob('*fits*') for file_ in fits_files: os.remove(file_) # if time interval is not defined, set the relevant 1 month interval if start is None or stop is None: [start, stop] = set_interval() start = DateTime(start).date stop = DateTime(stop).date # extract acis and hrc evt2 files for inst in ('acis', 'hrc'): run_arc(inst, start, stop)
def test_interpolate_msid_times(): start = '2008:002:21:48:00' stop = '2008:002:21:50:00' dat = fetch.MSID('aoattqt1', start, stop) dt = 10.0 times = dat.tstart + np.arange((dat.tstop - dat.tstart) // dt + 3) * dt dat.interpolate(times=times) assert np.allclose(dat.vals, [ -0.33072634, -0.33072637, -0.33072674, -0.33072665, -0.33073477, -0.330761, -0.33080694, -0.33089434, -0.33089264, -0.33097442, -0.33123678 ]) assert np.all(DateTime(dat.times).date == DATES_EXPECT3) dat = fetch.MSID('aogyrct1', start, stop) dat.interpolate(times=times) assert np.all(dat.vals == [ -22247, -21117, -19988, -18839, -17468, -15605, -13000, -9360, -4052, 2752, 10648 ]) assert np.all(DateTime(dat.times).date == DATES_EXPECT3) dat = fetch.MSID('aopcadmd', start, stop) dat.interpolate(times=times) assert np.all(dat.vals == [ 'NPNT', 'NPNT', 'NMAN', 'NMAN', 'NMAN', 'NMAN', 'NMAN', 'NMAN', 'NMAN', 'NMAN', 'NMAN' ]) assert np.all(DateTime(dat.times).date == DATES_EXPECT3)
def main(): """ Program-level main. This will normally be called via the update_archive.py wrapper script installed in $SKA/share/eng_archive. """ # Allow for a cmd line option --date-start. If supplied then loop the # effective value of opt.date_now from date_start to the cmd line # --date-now in steps of --max-lookback-time if opt.date_start is None: date_nows = [opt.date_now] else: t_starts = np.arange( DateTime(opt.date_start).secs, DateTime(opt.date_now).secs, opt.max_lookback_time * 86400.) date_nows = [DateTime(t).date for t in t_starts] date_nows.append(opt.date_now) # Drop the first date_now because that is covered by the second entry # minus the extended max_lookback_time (below). date_nows = date_nows[1:] # Increase max_lookback_time by 50%, but by no less than 2 days and no more than # 10 days. opt.max_lookback_time += min(max(opt.max_lookback_time * 0.5, 2), 10) for date_now in date_nows: opt.date_now = date_now main_loop()
def test_stop_day(self): self.assertEqual( DateTime('1996365.010203').day_end().iso, '1996-12-31 00:00:00.000') self.assertEqual( DateTime('1996366.010203').day_end().iso, '1997-01-01 00:00:00.000')
def make_archfiles_db(filename, content_def): # Do nothing if it is already there if os.path.exists(filename): return datestart = DateTime(DateTime(opt.start).secs - 60) tstart = datestart.secs tstop = tstart year, doy = datestart.date.split(':')[:2] times, indexes = derived.times_indexes(tstart, tstop, content_def['time_step']) logger.info('Creating db {}'.format(filename)) archfiles_def = open(Path(__file__).parent / 'archfiles_def.sql').read() db = Ska.DBI.DBI(dbi='sqlite', server=filename) db.execute(archfiles_def) archfiles_row = dict( filename='{}:0:1'.format(content_def['content']), filetime=0, year=year, doy=doy, tstart=tstart, tstop=tstop, rowstart=0, rowstop=0, startmjf=indexes[0], # really index0 stopmjf=indexes[-1], # really index1 date=datestart.date) db.insert(archfiles_row, 'archfiles')
def save_snapshot(self, fit_stat=None, method=None): """Save a snapshot of fit statistics. :param fit_stat: Manual way to pass fit statistic (may not be necessary in future) :param method: Manual way to pass method (may not be necessary in future) """ if fit_stat is None: fit_stat = self.model.calc_stat() pattern = 'Final fit statistic\s*=\s*([0-9.e]+\+?\d*)\D+(\d+)' found = re.findall(pattern, self.sherpa_log_capture_string.getvalue()) if found: final_eval_num = found[-1][-1] else: final_eval_num = None snapshot = {} for pars in self.model.pars: snapshot[pars['full_name']] = { k: pars[k] for k in ('frozen', 'min', 'max', 'val') } snapshot['fit_stat'] = fit_stat snapshot['final_eval_num'] = final_eval_num snapshot['tstart'] = DateTime(self.model.tstart).date snapshot['tstop'] = DateTime(self.model.tstop).date snapshot['method'] = method snapshot['date'] = DateTime().date self.snapshots.append(snapshot)
def mups_2_temps_xout_2(): close('all') t1 = '2013:231:11:00:00' t2 = '2013:231:12:45:00' msids = ['AOVBM2FS', 'PM2THV1T', 'PM2THV2T'] data = fetch.Msidset(msids, t1, t2, stat=None) b2_exp = data['PM2THV1T'].vals + 3 xticks = np.linspace(DateTime(t1).secs, DateTime(t2).secs, 11) xticklabels = [DateTime(t).date[5:17] for t in xticks] fig = plt.figure(figsize=[18, 9], facecolor='w') rect = [0.06, 0.15, 0.88, 0.75] ax1 = fig.add_axes(rect) ax1.plot(data['AOVBM2FS'].times, data['AOVBM2FS'].raw_vals, color=[0.4, 0.4, 0.4]) ax1.set_ylim(-1, 12) ax1.set_yticks([0]) ax1.set_yticklabels(['AOVBM2FS'], rotation=45) for t in ax1.yaxis.get_ticklines(): t.set_visible(False) ax1.set_xticks(xticks) ax1.set_xticklabels(xticklabels, rotation=45, ha='right') ax1.set_xlim(xticks[0], xticks[-1]) ax2 = fig.add_axes(rect, frameon=False) ax2.plot(data['PM2THV1T'].times, data['PM2THV1T'].vals, color="#56B4E9", linewidth=4.0, label='2A Actual') ax2.plot(data['PM2THV2T'].times, data['PM2THV2T'].vals, 'm', linewidth=4.0, label='2B Actual') ax2.plot(data['PM2THV1T'].times, b2_exp, 'm:', linewidth=4.0, label='2B Predicted w/o Firings') ax2.legend(loc='upper left') ax2.set_ylim(100, 150) ax2.yaxis.set_label_position('right') ax2.yaxis.tick_right() ax2.set_ylabel('MUPS-2 Temperatures [deg F]') ax2.set_xticks(xticks) ax2.set_xticklabels('') ax2.set_xlim(xticks[0], xticks[-1]) title('MUPS B-Side Activations and MUPS 2B Temperature - 2013:231 Firing') fig.savefig('2013_231_mups2_temps_expected.png')
def __init__(self, msids, start, stop): super(MSIDset, self).__init__() self.tstart = DateTime(start).secs self.tstop = DateTime(stop).secs self.datestart = DateTime(self.tstart).date self.datestop = DateTime(self.tstop).date slot_datas = {} slots = set(slot_for_msid(confirm_msid(msid)) for msid in msids) for slot in slots: # get the 8x8 data tstop = self.tstop + 33.0 # Major frame of padding slot_data = aca_l0.get_slot_data( self.tstart, tstop, slot, imgsize=[8], columns=ACA_DTYPE_NAMES) # Find samples where the time stamp changes by a value other than 4.1 secs # (which is the value for 8x8 readouts). In that case there must have been a # break in L0 decom, typically due to a change to 4x4 or 6x6 data. # t[0] = 1.0 # t[1] = 5.1 <= This record could be bad, as indicated by the gap afterward # t[2, 3] = 17.4, 21.5 # To form the time diffs first add `tstop` to the end so that if 8x8 data # does not extend through `tstop` then the last record gets chopped. dt = np.diff(np.concatenate([slot_data['TIME'], [tstop]])) bad = np.abs(dt - 4.1) > 1e-3 slot_data[bad] = ma.masked # Chop off the padding i_stop = np.searchsorted(slot_data['TIME'], self.tstop, side='right') slot_data = slot_data[:i_stop] # explicitly unmask useful columns slot_data['TIME'].mask = ma.nomask slot_data['IMGSIZE'].mask = ma.nomask slot_data['FILENAME'].mask = ma.nomask slot_datas[slot] = slot_data # make a shared time ndarray that is the union of the time sets in the # slots. The ACA L0 telemetry has the same timestamps across slots, # so the only differences here are caused by different times in # non-TRAK across the slots (usually SRCH differences at the beginning # of the observation) shared_time = np.unique(np.concatenate([ slot_datas[slot]['TIME'].data for slot in slots])) for msid in msids: hdr3_msid = confirm_msid(msid) slot = slot_for_msid(hdr3_msid) full_data = ma.zeros(len(shared_time), dtype=slot_datas[slot].dtype) full_data.mask = ma.masked fd_idx = search_both_sorted(shared_time, slot_datas[slot]['TIME']) full_data[fd_idx] = slot_datas[slot] # make a data dictionary to feed to the MSID constructor slot_data = {'vals': HDR3_DEF[hdr3_msid]['value'](full_data), 'desc': HDR3_DEF[hdr3_msid]['desc'], 'longdesc': HDR3_DEF[hdr3_msid]['longdesc'], 'times': shared_time, 'hdr3_msid': hdr3_msid} self[msid] = MSID(msid, start, stop, slot_data)
def check_content(outdir, content, msids=None): outdir = Path(outdir) if outdir.exists(): shutil.rmtree(outdir) print() print(f'Test dir: {outdir}') if msids is None: msids = CONTENTS[content] basedir_ref = outdir / 'orig' basedir_test = outdir / 'test' basedir_ref.mkdir(parents=True) basedir_test.mkdir(parents=True) # Make a local hard-link copy of select parts (content and msids) of the # "official" cheta archive data (nominally $SKA/data/engarchive) in basedir_ref. # This hard-link repo servers as the source for making the sync repo so this # is faster/lighter. make_linked_local_archive(basedir_ref, content, msids) # Make the sync repo, using basedir_ref as input data and outputting the # sync/ dir to basedir_test. with set_fetch_basedir(basedir_ref): make_sync_repo(basedir_test, content) # Make stubs of archive content, meaning filled with mostly zeros until about # before before test start date, then some real data to get the sync'ing going. make_stub_content(content, date=DateTime(START) - 2, basedir_stub=basedir_test, basedir_ref=basedir_ref, msids=msids) date_stop = (DateTime(STOP) + 2).date print(f'Updating client archive {content}') with set_fetch_basedir(basedir_test): update_client_archive.main([f'--content={content}', f'--log-level={LOG_LEVEL}', f'--date-stop={date_stop}', f'--data-root={basedir_test}', f'--sync-root={basedir_test}']) print(f'Checking {content} {msids}') for stat in None, '5min', 'daily': for msid in msids: fetch.times_cache['key'] = None with set_fetch_basedir(basedir_test): dat_stub = fetch.Msid(msid, START, STOP, stat=stat) fetch.times_cache['key'] = None with set_fetch_basedir(basedir_ref): dat_orig = fetch.Msid(msid, START, STOP, stat=stat) for attr in dat_orig.colnames: assert np.all(getattr(dat_stub, attr) == getattr(dat_orig, attr))
def __init__(self, msid, slot, start, stop): self.tstart = DateTime(start).secs self.tstop = DateTime(stop).secs self.datestart = DateTime(self.tstart).date self.datestop = DateTime(self.tstop).date self.slot = slot self._check_msid(msid) self._get_data()
def __init__(self, msids, start, stop): super(MSIDset, self).__init__() self.tstart = DateTime(start).secs self.tstop = DateTime(stop).secs self.datestart = DateTime(self.tstart).date self.datestop = DateTime(self.tstop).date for msid in msids: self[msid] = MSID(msid, self.tstart, self.tstop)
def get_trak_cat_from_telem(start, stop, cmd_quat): start = DateTime(start) stop = DateTime(stop) msids = [ "{}{}".format(m, i) for m in ['AOACYAN', 'AOACZAN', 'AOACFID', 'AOIMAGE', 'AOACFCT'] for i in range(0, 8) ] telem = fetch.MSIDset( ['AOACASEQ', 'CORADMEN', 'AOPCADMD', 'AONSTARS', 'AOKALSTR'] + msids, start, stop) att = fetch.MSIDset(['AOATTQT{}'.format(i) for i in [1, 2, 3, 4]], start, stop) cat = {} for slot in range(0, 8): track = telem['AOACFCT{}'.format(slot)].vals == 'TRAK' fid = telem['AOACFID{}'.format(slot)].vals == 'FID ' star = telem['AOIMAGE{}'.format(slot)].vals == 'STAR' n = 30 if np.count_nonzero(track) < n: continue if np.any(fid & track): cat[slot] = { 'type': 'FID', 'yag': telem['AOACYAN{}'.format(slot)].vals[fid & track][0], 'zag': telem['AOACZAN{}'.format(slot)].vals[fid & track][0] } else: n_samples = np.count_nonzero(track & star) if n_samples < (n + 4): continue # If there is tracked data with a star, let's try to get our n samples from about # the middle of the range mid_point = int(n_samples / 2.) yags = [] zags = [] for sample in range(mid_point - int(n / 2.), mid_point + int(n / 2.)): qref = Quat( normalize([ att['AOATTQT{}'.format(i)].vals[track & star][sample] for i in [1, 2, 3, 4] ])) ra, dec = yagzag2radec( telem['AOACYAN{}'.format(slot)].vals[track & star][sample] / 3600., telem['AOACZAN{}'.format(slot)].vals[track & star][sample] / 3600., qref) yag, zag = radec2yagzag(ra, dec, cmd_quat) yags.append(yag) zags.append(zag) # This doesn't detect MON just yet cat[slot] = { 'type': 'STAR', 'yag': np.median(yags) * 3600., 'zag': np.median(zags) * 3600. } return cat, telem
def write_report(thermal_msid_checks_file, t1, t2): """ Write Thermal Weekly Report :param thermal_msid_checks_file: Name of pickle file listing msids and related information :param t1: string containing start time in HOSC format :param t2: string containign stop time in HOSC format Note, in the past: thermal_msid_checks_file = 'thermalmsiddata.pkl' """ thermdict, missing, notinarchive = pickle.load( open(thermal_msid_checks_file, 'r')) t1 = DateTime(t1).date t2 = DateTime(t2).date dayrange = (t1[:9] + '-' + t2[:9]).replace(':', '') power = get_average_tel_power(t1, t2) eclfile = pathjoin(AXAFDATA, 'ECLIPSE.txt') ecltext = get_eclipse_text(eclfile, t1, t2) allviolations, missingmsids, checkedmsids = check_violations( thermdict, t1, t2) # 3shtren and 4csdhav are not decommed correctly in the CXC archive if '3shtren' in allviolations.keys(): _ = allviolations.pop('3shtren') if '4csdhav' in allviolations.keys(): _ = allviolations.pop('4csdhav') limitchanges = check_limit_changes(t1, t2) html_limit_change_table = 'None' env = ja.Environment(loader=ja.FileSystemLoader( home + '/AXAFLIB/thermal_weekly_report/thermal_weekly_report/templates')) template = env.get_template('thermal_weekly_template.htm') webpage = template.render(startday=t1, endday=t2, dayrange=dayrange, power=str('%5.1f' % power), eclipse=ecltext, violations=allviolations, limitchanges=limitchanges) reportfilename = 'THERMAL_Weekly_' + dayrange + '.htm' outfile = file(reportfilename, 'w+') outfile.writelines(webpage) outfile.close() print(' Saved weekly report to {0}\n'.format(reportfilename)) return reportfilename
def update(self): """ Retrieve ACA0 telemetry files from the CXC archive, store in the Ska/ACA archive, and update database of files. """ contentdir = self.data_root if not os.path.exists(contentdir): os.makedirs(contentdir) if not os.path.exists(self.temp_root): os.makedirs(self.temp_root) archdb = os.path.join(contentdir, 'archfiles.db3') # if the database of the archived files does not exist, # or is empty, make it if not os.path.exists(archdb) or os.stat(archdb).st_size == 0: logger.info("creating archfiles db from %s" % self.sql_def) db_sql = os.path.join(os.environ['SKA_DATA'], 'mica', self.sql_def) db_init_cmds = file(db_sql).read() with Ska.DBI.DBI(**self.db) as db: db.execute(db_init_cmds, commit=True) if self.start: datestart = DateTime(self.start) else: # Get datestart as the most-recent file time from archfiles table # will need min-of-max-slot-datestart with Ska.DBI.DBI(**self.db) as db: last_time = min([db.fetchone( "select max(filetime) from archfiles where slot = %d" % s)['max(filetime)'] for s in range(0, 8)]) if last_time is None: raise ValueError( "No files in archive to do update-since-last-run mode.\n" + "Please specify a time with --start") datestart = DateTime(last_time) datestop = DateTime(self.stop) padding_seconds = 10000 # loop over the specified time range in chunks of # days_at_once in seconds with some padding for tstart in np.arange(datestart.day_start().secs, datestop.day_end().secs, self.days_at_once * 86400): # set times for a chunk range_tstart = tstart - padding_seconds range_tstop = tstart + self.days_at_once * 86400 if range_tstop > datestop.day_end().secs: range_tstop = datestop.day_end().secs range_tstop += padding_seconds # make a temporary directory tmpdir = Ska.File.TempDir(dir=self.temp_root) dirname = tmpdir.name logger.debug("Files save to temp dir %s" % dirname) # get the files, store in file archive, and record in database with Ska.File.chdir(dirname): fetched_files = self._fetch_by_time(range_tstart, range_tstop) self._insert_files(fetched_files) timestamp_file = os.path.join(self.data_root, 'last_timestamp.txt') # get list of missing files since the last time the tool ingested # files. If this is first run of the tool, check from the start of # the requested time range if (os.path.exists(timestamp_file) and os.stat(timestamp_file).st_size > 0): cda_checked_timestamp = open(timestamp_file).read().rstrip() else: cda_checked_timestamp = DateTime(self.start).date missing_datetime = DateTime(cda_checked_timestamp) missing_files, last_ingest_date = \ self._get_missing_archive_files(missing_datetime, only_new=True) # update the file to have up through the last confirmed good file # even before we try to fetch missing ones open(timestamp_file, 'w').write("%s" % last_ingest_date) if len(missing_files): logger.info("Found %d missing individual files" % len(missing_files)) # make a temporary directory tmpdir = Ska.File.TempDir(dir=self.temp_root) dirname = tmpdir.name logger.info("File save to temp dir %s" % dirname) with Ska.File.chdir(dirname): fetched_files, ingest_times = \ self._fetch_individual_files(missing_files) self._insert_files(fetched_files) last_ingest_date = missing_files[-1]['ingest_date'] # update the file to have up through the last confirmed good file # even before we try to fetch missing ones open(timestamp_file, 'w').write("%s" % last_ingest_date) else: logger.info("No missing files")