def compute_mag_qdc_t(st): '''Compute QDC time. For the 4th or later in the month load the previous month, otherwise go back two months. This gives a few days for data to be transferred and QDCs to be made and checked.''' qdc_t = dt64.get_start_of_previous_month(st) if dt64.get_day_of_month(st) < 4: qdc_t = dt64.get_start_of_previous_month(qdc_t) return qdc_t
def make_aurorawatch_plot(project, site, st, et, rolling, exif_tags): ''' Load data and make the AuroraWatch activity plot. Plots always cover 24 hours, but may begin at midnight for day plots, or at any other hour for rolling plots. This function uses the previous 72 hours to help fit the quiet-day curve. project: name of project site: name of site st: start time. For day plots this is the start of the day. For rolling plots this is the start of the rolling 24 hour period. et: end time. For day plots this is the start of the following day. For rolling plots it is the end of the 24 hour period. rolling: flag to indicate if rolling plot should also be made. It is not otherwise possible to identify rolling plots which start at midnight. ''' # global mag_fstr global args # Export to global names for debugging global mag_data global mag_qdc global activity day = np.timedelta64(24, 'h') archive, archive_details = ap.get_archive_info(project, site, 'MagData') # Load the data to plot. For rolling plots load upto midnight so # that both the rolling plot and the current day plot can be # generated efficiently. mag_data = my_load_data(project, site, 'MagData', st, dt64.ceil(et, day)) if mag_data is None or \ not np.any(np.logical_not(np.isnan(mag_data.data))): # not .np.any(etc) eliminates empty array or array of just nans logger.info('No magnetic field data') return # Load up some data from previous days to and apply a # least-squares fit to remove baseline drifts. Data from the # current day is not used. This ensures that results do not change # over the current day when new data becomes available. qdc_fit_interval = args.qdc_fit_interval * day fit_et = dt64.ceil(st, day) # Could be doing a rolling plot fit_st = fit_et - qdc_fit_interval fit_data = my_load_data(project, site, 'MagData', fit_st, fit_et) # Load a QDC. For the 4th or later in the month load the previous # month, otherwise go back two months. This gives a few days for # data to be transferred, and QDCs to be made and checked. qdc_t = dt64.get_start_of_previous_month(st) if dt64.get_day_of_month(st) < 4: qdc_t = dt64.get_start_of_previous_month(qdc_t) mag_qdc = ap.magdata.load_qdc(project, site, qdc_t, tries=6) if mag_qdc is None: logger.info('No QDC') elif fit_data is None: # Cannot fit, so assume no errors in QDC errors = [0.0] else: try: # Fit the QDC to the previous data qdc_aligned, errors, fi = mag_qdc.align(\ fit_data, fit=ap.data.Data.minimise_sign_error_fit, plot_fit=args.plot_fit, full_output=True) except Exception as e: logger.warn('Could not fit QDC') logger.info(str(e)) errors = [0.0] else: # Fitted ok, plot if necessary if args.plot_fit: fig = plt.gcf() fig.set_figwidth(6.4) fig.set_figheight(4.8) fig.subplots_adjust(bottom=0.1, top=0.85, left=0.15, right=0.925) fit_fstr = mag_fstr[:(mag_fstr.rindex('.'))] + '_fit.png' mysavefig(fig, dt64.strftime(dt64.ceil(st, day), fit_fstr), exif_tags) # Adjust the quiet day curve with the error obtained by fitting to # previous days. if mag_qdc is None: mag_qdc_adj = None else: mag_qdc_adj = copy.deepcopy(mag_qdc) mag_qdc_adj.data -= errors[0] # Ensure data gaps are marked as such in the plots. Straight lines # across large gaps look bad! mag_data = mag_data.mark_missing_data(cadence=2*mag_data.nominal_cadence) # Do day plot. Trim start time for occasions when making a day # plot simultaneously with a rolling plot. st2 = dt64.ceil(st, day) md_day = mag_data.extract(start_time=st2) act_ki = activity_plot(md_day, mag_qdc_adj, dt64.strftime(st2, mag_fstr), exif_tags, k_index_filename=dt64.strftime(st2, k_fstr)) r = [md_day] r.extend(act_ki) if rolling: # Trim end time md_rolling = mag_data.extract(end_time=et) act_ki_rolling = activity_plot(md_rolling, mag_qdc_adj, rolling_magdata_filename, exif_tags, k_index_filename=rolling_k_filename) r.append(md_rolling) r.extend(act_ki_rolling) return r
args = parser.parse_args() if __name__ == '__main__': logging.basicConfig(level=getattr(logging, args.log_level.upper()), format=args.log_format) # Use a consistent value for current time day = np.timedelta64(1, 'D').astype('m8[us]') now = np.datetime64('now', 'us') today = dt64.floor(now, day) yesterday = today - day tomorrow = today + day if args.start_time is None: start_time = dt64.get_start_of_previous_month(today) elif args.start_time == 'today': start_time = today elif args.start_time == 'yesterday': start_time = yesterday else: start_time = dt64.floor(np.datetime64(args.start_time), day) if args.end_time is None: end_time = start_time + day elif args.end_time == 'today': end_time = today elif args.end_time == 'yesterday': end_time = yesterday elif args.end_time == 'tomorrow': end_time = tomorrow
def load_qdc(project, site, time, archive=None, channels=None, path=None, tries=1, realtime=False, load_function=None, full_output=False): '''Load quiet-day curve. project: name of the project (upper case) site: site abbreviation (upper case) time: a time within the quiet-day curve period The following optional parameters are recognised: archive: name of the archive. Required if more than one archive is present and there is not an archive called "default". channels: data channel(s) to load. All are loaded if not specified. tries: The number of attempts to load a quiet-day curve. If >1 and the first attempt is not successful then an attempt will be made to load the previous QDC. path: URL or file path, specified as a strftime format specifier. Alternatively can be a function reference which is passed the time and returns the filename. If given this overrides the standard load path. load_function: Pass responsibility for loading the data to the given function reference, after validating the input parameters. ''' data_type = 'MagQDC' archive, ad = ap.get_archive_info(project, site, data_type, archive=archive) if channels is not None: # Ensure it is a 1D numpy array channels = np.array(channels).flatten() for c in channels: if c not in ad['channels']: raise ValueError('Unknown channel (%s)' % str(c)) else: channels = ad['channels'] if path is None: path = ad['path'] if load_function is None: load_function = ad.get('load_function', None) if tries is None: tries = 1 if load_function: # Pass responsibility for loading to some other # function. Parameters have already been checked. return load_function(project, site, data_type, time, archive=archive, channels=channels, path=path, tries=tries, realtime=realtime, full_output=full_output) data = [] t = dt64.get_start_of_month(time) if realtime: # For realtime use the QDC for the month is (was) not # available, so use the previous month's QDC t = dt64.get_start_of_previous_month(t) # Early in the month the previous motnh's QDC was probably not # computed, so use the month before that qdc_rollover_day = ad.get('qdc_rollover_day', 4) if dt64.get_day_of_month(time) < qdc_rollover_day: t = dt64.get_start_of_previous_month(t) for n in range(tries): try: if hasattr(path, '__call__'): # Function: call it with relevant information to get the path file_name = path(t, project=project, site=site, data_type=data_type, archive=archive, channels=channels) else: file_name = dt64.strftime(t, path) logger.info('loading ' + file_name) r = ad['load_converter'](file_name, ad, project=project, site=site, data_type=data_type, start_time=np.timedelta64(0, 'h'), end_time=np.timedelta64(24, 'h'), archive=archive, channels=channels, path=path) if r is not None: r.extract(inplace=True, channels=channels) if full_output: r2 = {'magqdc': r, 'tries': n + 1, 'maxtries': tries} return r2 else: return r finally: # Go to start of previous month t = dt64.get_start_of_month(t - np.timedelta64(24, 'h')) return None