def sync_stat_archive(opt, msid_files, logger, content, stat, index_tbl): """ Sync the archive for ``content``. This calls _sync_stat_archive with a wrapper to catch an exception related to the last_date_id file being out of sync. :param opt: :param msid_files: :param logger: :param content: :param stat: stat interval '5min' or 'daily' :param index_tbl: table of sync file entries :return: """ try: _sync_stat_archive(opt, msid_files, logger, content, stat, index_tbl) except RowMismatchError as err: pth = Path(fetch.msid_files['last_date_id'].abs) if pth.exists(): msg = f'File {pth} was out of sync with stats archive, generating exception: {err}' logger.warn(msg) logger.warn(f'Attempting to fix by removing that file and trying to sync again.') pth.unlink() _sync_stat_archive(opt, msid_files, logger, content, stat, index_tbl) process_errors.append( f'WARNING: file {pth} was out of sync with stats archive, fixed by deleting it')
def get_stats_over_time(start, stop=None, sp=False, dp=None, ir=False, ms=None, slots='combined', t_samp=1000): """ Equivalent to get_stats_per_interval, but concatenate the results for all obsids within the specified time interval. """ # Get obsids in time range and collect all the per-interval statistics obsids = events.obsids.filter(start, stop, dur__gt=2000) stats_list = [] for obsid in obsids: set_FILES_context(obsid.obsid, sp, dp, ir, ms, t_samp, slots) # First check that there is the raw dat file for this obsid. Nothing # can be done without this. dat_file = FILES['dat.pkl'].rel if not os.path.exists(dat_file): logger.info('Skipping {}: {} not in archive'.format(obsid, dat_file)) continue # Now get the stats for this obsid. Hopefully it has already been computed and # is cached as a file. If not, try to compute the stats (and cache). If that # fails then press on but touch a file to indicate failure so subsequent attempts # don't bother. logger.info('Processing obsid {}'.format(obsid)) try: stats = get_cached_stats() # depends on the context set previously except FailedStatsFile: # Previously failed logger.info(' Skipping {}: failed statistics'.format(obsid.obsid)) continue except NoStatsFile: logger.info(' Reading pickled data file {}'.format(dat_file)) dat = pickle.load(open(dat_file, 'r')) try: logger.info(' Computing statistics') if slots == 'combined': stats = get_stats_per_interval_combined(dat, sp, dp, ir, ms, t_samp) else: stats = get_stats_per_interval_per_slot(dat, sp, dp, ir, ms, slots, t_samp) except ValueError as err: open(FILES['stats.ERR'].rel, 'w') # touch file to indicate failure to compute stats logger.warn(' ERROR: {}'.format(err)) stats['obsid'] = obsid.obsid stats_list.append(stats) stats = {} for case in STAT_CASES: stats[case] = {} for stat_type in STAT_TYPES: stats[case][stat_type] = np.hstack([x[case][stat_type] for x in stats_list]) # Set corresponding array of obsids for back-tracing outliers etc stats['obsid'] = np.hstack([np.ones(len(x['obc']['std']), dtype=int) * x['obsid'] for x in stats_list]) return stats
def get_kalman_predicted_over_time(start, stop=None, sp=False, dp=None, ir=False, ms=None): obsids = events.obsids.filter(start, stop, dur__gt=2000) tlm_durs = [] pred_durs = [] for obsid in obsids: logger.info('Reading data for obsid {}'.format(obsid)) try: dat = get_obsid_data(obsid.obsid) except Exception as err: logger.warn('Failed: {}'.format(err)) continue tlm_drops, pred_drops = get_kalman_predicted(dat, sp, dp, ir, ms)[-2:] tlm_durs.append(tlm_drops['duration']) pred_durs.append(pred_drops['duration']) tlm_durs = np.concatenate(tlm_durs) pred_durs = np.concatenate(pred_durs) return tlm_durs, pred_durs
def update_observed_aimpoints(): """ Update the ``OBSERVED_AIMPOINTS_FILE`` table (ascii ECSV format) in place to reflect information about observed aimpoints, and in particular the delta offset from the planned value. """ # Default is between NOW and NOW - 14 days start = DateTime(opt.start) - (14 if opt.start is None else 0) stop = DateTime(opt.stop) # Get science obsids obsids = [evt.obsid for evt in events.obsids.filter(start, stop) if evt.obsid < 40000] # Read in existing file if it exists and make a set of already-processed obsids filename = os.path.join(opt.data_root, OBSERVED_AIMPOINTS_FILE) if os.path.exists(filename): logger.info('Reading {}'.format(filename)) dat_old = Table.read(filename, format='ascii.ecsv', guess=False) processed_obsids = set(dat_old['obsid']) else: dat_old = None processed_obsids = set() rows = [] for obsid in obsids: if obsid in processed_obsids: logger.info('Skipping obsid {}: already processed'.format(obsid)) continue try: vals = get_observed_aimpoint_offset(obsid) except NoObsidError: # not yet in archive logger.info('Skipping obsid {}: not in archive yet'.format(obsid)) continue except Exception as err: logger.info('ERROR: {}'.format(err)) continue logger.info('Obsid={obsid:5d} detector={detector:6s} ' 'chipx={chipx:.1f} chipy={chipy:.1f} dx={dx:.1f} dy={dy:.1f} dr={dr:.1f}' .format(**vals)) if abs(vals['dx']) > 10 or abs(vals['dy']) > 10: logger.warn('WARNING: large dx or dy') rows.append(vals) if rows: dat = Table(rows=rows, names=sorted(rows[0])) if dat_old is not None: dat = vstack([dat_old, dat]) logger.info('Writing {}'.format(filename)) for name in 'dr dx dy obs_chipx obs_chipy sim_z_off'.split(): dat[name].format = '.2f' dat.sort('mean_date') dat.write(filename, format='ascii.ecsv') else: dat = dat_old return dat
def update_observed_aimpoints(): """ Update the ``OBSERVED_AIMPOINTS_FILE`` table (ascii ECSV format) in place to reflect information about observed aimpoints, and in particular the delta offset from the planned value. """ # Default is between NOW and NOW - 14 days start = DateTime(opt.start) - (14 if opt.start is None else 0) stop = DateTime(opt.stop) # Get science obsids obsids = [ evt.obsid for evt in events.obsids.filter(start, stop) if evt.obsid < 40000 ] # Read in existing file if it exists and make a set of already-processed obsids filename = os.path.join(opt.data_root, OBSERVED_AIMPOINTS_FILE) if os.path.exists(filename): logger.info('Reading {}'.format(filename)) dat_old = Table.read(filename, format='ascii.ecsv', guess=False) processed_obsids = set(dat_old['obsid']) else: dat_old = None processed_obsids = set() rows = [] for obsid in obsids: if obsid in processed_obsids: logger.info('Skipping obsid {}: already processed'.format(obsid)) continue try: vals = get_observed_aimpoint_offset(obsid) except NoObsidError: # not yet in archive logger.info('Skipping obsid {}: not in archive yet'.format(obsid)) continue except Exception as err: logger.info('ERROR: {}'.format(err)) continue logger.info( 'Obsid={obsid:5d} detector={detector:6s} ' 'chipx={chipx:.1f} chipy={chipy:.1f} dx={dx:.1f} dy={dy:.1f} dr={dr:.1f}' .format(**vals)) if abs(vals['dx']) > 10 or abs(vals['dy']) > 10: logger.warn('WARNING: large dx or dy') rows.append(vals) if rows: dat = Table(rows=rows, names=sorted(rows[0])) if dat_old is not None: dat = vstack([dat_old, dat]) logger.info('Writing {}'.format(filename)) for name in 'dr dx dy obs_chipx obs_chipy sim_z_off'.split(): dat[name].format = '.2f' dat.sort('mean_date') dat.write(filename, format='ascii.ecsv', overwrite=True) else: dat = dat_old return dat
def update(EventModel, date_stop): import django.db from django.core.exceptions import ObjectDoesNotExist from .events import models date_stop = DateTime(date_stop) cls_name = EventModel.__name__ try: update = models.Update.objects.get(name=cls_name) except ObjectDoesNotExist: logger.info('No previous update for {} found'.format(cls_name)) duration = EventModel.lookback update = models.Update(name=cls_name, date=date_stop.date) date_start = date_stop - EventModel.lookback else: duration = date_stop - DateTime(update.date) date_start = DateTime(update.date) - EventModel.lookback update.date = date_stop.date # Some events like LoadSegment or DsnComm might change in the database after # having been ingested. Use lookback_delete (less than lookback) to # always remove events in that range and re-ingest. if duration >= 0.5 and hasattr(EventModel, 'lookback_delete'): delete_date = DateTime(update.date) - EventModel.lookback_delete delete_from_date(EventModel, delete_date, set_update_date=False) if duration < 0.5: logger.info( 'Skipping {} events because update duration={:.1f} is < 0.5 day'. format(cls_name, duration)) return # Some events like LoadSegment, DsnComm are defined into the future, so # modify date_stop accordingly. Note that update.date is set to the # nominal date_stop (typically NOW), and this refers more to the last date # of processing rather than the actual last date in the archive. if hasattr(EventModel, 'lookforward'): date_stop = date_stop + EventModel.lookforward logger.info('Updating {} events from {} to {}'.format( cls_name, date_start.date[:-4], date_stop.date[:-4])) # Get events for this model from telemetry. This is returned as a list # of dicts with key/val pairs corresponding to model fields. events_in_dates = EventModel.get_events(date_start, date_stop) # Determine which of the events is not already in the database and # put them in a list for saving. event_models, events = get_events_and_event_models(EventModel, cls_name, events_in_dates) # Save the new events in an atomic fashion with django.db.transaction.atomic(): for event, event_model in zip(events, event_models): try: # In order to catch an IntegrityError here and press on, need to # wrap this in atomic(). This was driven by bad data in iFOT, namely # duplicate PassPlans that point to the same DsnComm, which gives an # IntegrityError because those are related as one-to-one. with django.db.transaction.atomic(): save_event_to_database(cls_name, event, event_model, models) except django.db.utils.IntegrityError: import traceback logger.warn(f'WARNING: IntegrityError skipping {event_model}') logger.warn(f'Event dict:\n{event}') logger.warn(f'Traceback:\n{traceback.format_exc()}') continue # If processing got here with no exceptions then save the event update # information to database update.save()