def load_bad_data(project, site, data_type, start_time, end_time, archive=None, path=None, extension='.bad', **kwargs): '''Load data from bad data files, usually those which end in .bad. This function is a wrapper which calls aurorawatchnet.load_data() after appending an extension to the file names. The path cannot be a callable. ''' if path is None: ai = ap.get_archive_info(project, site, data_type, archive=archive) path = ai[1]['path'] + extension return ap.load_data(project, site, data_type, start_time, end_time, archive=ai[0], path=path, **kwargs)
def uit_path(t, project, site, data_type, archive, channels): if uit_password is None: raise Exception(__name__ + '.uit_password must be set, ' + 'to obtain a password see ' + 'http://flux.phys.uit.no/div/DataAccess.html') # Expand the path format string with the specific UIT variables, # including password. a, d = copy.deepcopy(ap.get_archive_info(project, site, data_type, archive=archive)) d['uit_password'] = uit_password fstr = path_fstr % d return dt64.strftime(t, fstr)
st = dt64.floor(st, day) et = dt64.ceil(et, day) for n in range(len(project_list)): project = project_list[n] site = site_list[n] # Get baseline archive to use for target data if project in baseline_archive and site in baseline_archive[project]: bl_archive = baseline_archive[project][site] else: bl_archive = 'realtime_baseline' an, ai = ap.get_archive_info(project, site, 'MagData', archive=bl_archive) if 'qdc_fit_duration' not in ai: logger.error('no qdc_fit_duration found in %s archive for %s/%s', an, project, site) continue if 'realtime_qdc' not in ai: logger.warning('realtime_qdc option not in %s archive for %s/%s', an, project, site) qdc_fit_duration = ai['qdc_fit_duration'] qdc_fit_offset = ai.get('qdc_fit_offset', -qdc_fit_duration/2 - 1.5*day) qdc_tries = ai.get('qdc_tries', 3) # Get mag data archive to use for source data
else: qdc_archive = None logger.debug('Processing %s/%s' % (project, site)) # Attempt to import missing projects if project not in ap.projects: try: import_module('auroraplot.datasets.' + project.lower()) except: pass ax = None archive, ad = ap.get_archive_info(project, site, 'MagData', \ archive=archive) qdc_archive, qdc_ad = ap.get_archive_info(project, site, 'MagQDC', \ archive=qdc_archive) # Tune start/end times to avoid requesting data outside of # operational period site_st = ap.get_site_info(project, site, 'start_time') if site_st is None or site_st < st: site_st = st else: site_st = dt64.floor(site_st, day) site_et = ap.get_site_info(project, site, 'end_time') if site_et is None or site_et > et: site_et = et else:
if args.single_qdc: t2 = end_time else: t2 = dt64.get_start_of_next_month(t1) if args.plot: mag_qdc = ap.magdata.load_qdc(project_uc, site_uc, t1) if mag_qdc is not None: lh = mag_qdc.plot(axes=ax) for h in lh: h.set_label(dt64.strftime(t1, '%Y-%m-%d')) ax = plt.gca() else: archive, ad = ap.get_archive_info(project_uc, site_uc, 'MagData', archive=getattr(args, 'archive')) mag_data = ap.load_data(project_uc, site_uc, 'MagData', t1, t2, archive=archive, raise_all=args.raise_all) if mag_data is not None: mag_qdc = mag_data.make_qdc(smooth=args.smooth) qdc_archive, qdc_ad \ = ap.get_archive_info(project_uc, site_uc, 'MagQDC') if args.single_qdc: filename = args.single_qdc else: filename = dt64.strftime(t1, qdc_ad['path']) p = os.path.dirname(filename)
archive = ap.parse_archive_selection(args.archive) else: archive = {} for n in range(len(project_list)): project = project_list[n] site = site_list[n] # Get archive to use for data if project in archive and site in archive[project]: archive = archive[project][site] else: archive = None dest_an, dest_ai = ap.get_archive_info(project, site, 'MagData', archive=archive) src_an, src_ai = ap.get_archive_info(project, site, 'MagData', archive='original_' + dest_an) src_path = src_ai['path'] dest_path = dest_ai['path'] print('src_ai ' + src_an) print(repr(src_ai)) print('dest_ai ' + dest_an) print(repr(dest_ai)) # Tune start/end times to avoid requesting data outside of # operational period site_st = ap.get_site_info(project, site, 'start_time') if site_st is None or site_st < st: site_st = st
def make_aurorawatch_plot(project, site, st, et, rolling, exif_tags): ''' Load data and make the AuroraWatch activity plot. Plots always cover 24 hours, but may begin at midnight for day plots, or at any other hour for rolling plots. This function uses the previous 72 hours to help fit the quiet-day curve. project: name of project site: name of site st: start time. For day plots this is the start of the day. For rolling plots this is the start of the rolling 24 hour period. et: end time. For day plots this is the start of the following day. For rolling plots it is the end of the 24 hour period. rolling: flag to indicate if rolling plot should also be made. It is not otherwise possible to identify rolling plots which start at midnight. ''' # global mag_fstr global args # Export to global names for debugging global mag_data global mag_qdc global activity day = np.timedelta64(24, 'h') archive, archive_details = ap.get_archive_info(project, site, 'MagData') # Load the data to plot. For rolling plots load upto midnight so # that both the rolling plot and the current day plot can be # generated efficiently. mag_data = my_load_data(project, site, 'MagData', st, dt64.ceil(et, day)) if mag_data is None or \ not np.any(np.logical_not(np.isnan(mag_data.data))): # not .np.any(etc) eliminates empty array or array of just nans logger.info('No magnetic field data') return # Load up some data from previous days to and apply a # least-squares fit to remove baseline drifts. Data from the # current day is not used. This ensures that results do not change # over the current day when new data becomes available. qdc_fit_interval = args.qdc_fit_interval * day fit_et = dt64.ceil(st, day) # Could be doing a rolling plot fit_st = fit_et - qdc_fit_interval fit_data = my_load_data(project, site, 'MagData', fit_st, fit_et) # Load a QDC. mag_qdc = ap.magdata.load_qdc(project, site, st, tries=6, realtime=True) if mag_qdc is None: logger.info('No QDC') elif fit_data is None: # Cannot fit, so assume no errors in QDC errors = [0.0] else: try: # Fit the QDC to the previous data qdc_aligned, errors, fi = mag_qdc.align(\ fit_data, fit=ap.data.Data.minimise_sign_error_fit, plot_fit=args.plot_fit, full_output=True) except Exception as e: logger.warn('Could not fit QDC') logger.info(str(e)) errors = [0.0] else: # Fitted ok, plot if necessary if args.plot_fit: fig = plt.gcf() fig.set_figwidth(6.4) fig.set_figheight(4.8) fig.subplots_adjust(bottom=0.1, top=0.85, left=0.15, right=0.925) fit_fstr = mag_fstr[:(mag_fstr.rindex('.'))] + '_fit.png' mysavefig(fig, dt64.strftime(dt64.ceil(st, day), fit_fstr), exif_tags) # Adjust the quiet day curve with the error obtained by fitting to # previous days. if mag_qdc is None: mag_qdc_adj = None else: mag_qdc_adj = copy.deepcopy(mag_qdc) mag_qdc_adj.data -= errors[0] # Ensure data gaps are marked as such in the plots. Straight lines # across large gaps look bad! mag_data = mag_data.mark_missing_data(cadence=2*mag_data.nominal_cadence) # Do day plot. Trim start time for occasions when making a day # plot simultaneously with a rolling plot. st2 = dt64.ceil(st, day) md_day = mag_data.extract(start_time=st2) act_ki = activity_plot(md_day, mag_qdc_adj, dt64.strftime(st2, mag_fstr), exif_tags, k_index_filename=dt64.strftime(st2, k_fstr)) r = [md_day] r.extend(act_ki) if rolling: # Trim end time md_rolling = mag_data.extract(end_time=et) act_ki_rolling = activity_plot(md_rolling, mag_qdc_adj, rolling_magdata_filename, exif_tags, k_index_filename=rolling_k_filename) r.append(md_rolling) r.extend(act_ki_rolling) return r
archive = ap.parse_archive_selection(args.archive) else: archive = {} for n in range(len(project_list)): project = project_list[n] site = site_list[n] # Get archive to use for data if project in archive and site in archive[project]: archive = archive[project][site] else: archive = None dest_an, dest_ai = ap.get_archive_info(project, site, 'MagData', archive=archive) src_an, src_ai = ap.get_archive_info(project, site, 'MagData', archive='original_' + dest_an) src_path = src_ai['path'] dest_path = dest_ai['path'] print('src_ai ' + src_an) print(repr(src_ai)) print('dest_ai ' + dest_an) print(repr(dest_ai)) # Tune start/end times to avoid requesting data outside of # operational period
baseline_archive = {} st = dt64.floor(st, day) et = dt64.ceil(et, day) for n in range(len(project_list)): project = project_list[n] site = site_list[n] # Get baseline archive to use for target data if project in baseline_archive and site in baseline_archive[project]: bl_archive = baseline_archive[project][site] else: bl_archive = 'realtime_baseline' an, ai = ap.get_archive_info(project, site, 'MagData', archive=bl_archive) if 'qdc_fit_duration' not in ai: logger.error('no qdc_fit_duration found in %s archive for %s/%s', an, project, site) continue if 'realtime_qdc' not in ai: logger.warning('realtime_qdc option not in %s archive for %s/%s', an, project, site) qdc_fit_duration = ai['qdc_fit_duration'] qdc_fit_offset = ai.get('qdc_fit_offset', -qdc_fit_duration / 2 - 1.5 * day) qdc_tries = ai.get('qdc_tries', 3)
qdc_archive = qdc_archive[project][site] else: qdc_archive = None logger.debug('Processing %s/%s' % (project, site)) # Attempt to import missing projects if project not in ap.projects: try: import_module('auroraplot.datasets.' + project.lower()) except: pass ax = None archive, ad = ap.get_archive_info(project, site, 'MagData', \ archive=archive) qdc_archive, qdc_ad = ap.get_archive_info(project, site, 'MagQDC', \ archive=qdc_archive) # Tune start/end times to avoid requesting data outside of # operational period site_st = ap.get_site_info(project, site, 'start_time') if site_st is None or site_st < st: site_st = st else: site_st = dt64.floor(site_st, day) site_et = ap.get_site_info(project, site, 'end_time') if site_et is None or site_et > et: site_et = et else:
def load_qdc(project, site, time, archive=None, channels=None, path=None, tries=1, realtime=False, load_function=None, full_output=False): '''Load quiet-day curve. project: name of the project (upper case) site: site abbreviation (upper case) time: a time within the quiet-day curve period The following optional parameters are recognised: archive: name of the archive. Required if more than one archive is present and there is not an archive called "default". channels: data channel(s) to load. All are loaded if not specified. tries: The number of attempts to load a quiet-day curve. If >1 and the first attempt is not successful then an attempt will be made to load the previous QDC. path: URL or file path, specified as a strftime format specifier. Alternatively can be a function reference which is passed the time and returns the filename. If given this overrides the standard load path. load_function: Pass responsibility for loading the data to the given function reference, after validating the input parameters. ''' data_type = 'MagQDC' archive, ad = ap.get_archive_info(project, site, data_type, archive=archive) if channels is not None: # Ensure it is a 1D numpy array channels = np.array(channels).flatten() for c in channels: if c not in ad['channels']: raise ValueError('Unknown channel (%s)' % str(c)) else: channels = ad['channels'] if path is None: path = ad['path'] if load_function is None: load_function = ad.get('load_function', None) if tries is None: tries = 1 if load_function: # Pass responsibility for loading to some other # function. Parameters have already been checked. return load_function(project, site, data_type, time, archive=archive, channels=channels, path=path, tries=tries, realtime=realtime, full_output=full_output) data = [] t = dt64.get_start_of_month(time) if realtime: # For realtime use the QDC for the month is (was) not # available, so use the previous month's QDC t = dt64.get_start_of_previous_month(t) # Early in the month the previous motnh's QDC was probably not # computed, so use the month before that qdc_rollover_day = ad.get('qdc_rollover_day', 4) if dt64.get_day_of_month(time) < qdc_rollover_day: t = dt64.get_start_of_previous_month(t) for n in range(tries): try: if hasattr(path, '__call__'): # Function: call it with relevant information to get the path file_name = path(t, project=project, site=site, data_type=data_type, archive=archive, channels=channels) else: file_name = dt64.strftime(t, path) logger.info('loading ' + file_name) r = ad['load_converter'](file_name, ad, project=project, site=site, data_type=data_type, start_time=np.timedelta64(0, 'h'), end_time=np.timedelta64(24, 'h'), archive=archive, channels=channels, path=path) if r is not None: r.extract(inplace=True, channels=channels) if full_output: r2 = {'magqdc': r, 'tries': n + 1, 'maxtries': tries} return r2 else: return r finally: # Go to start of previous month t = dt64.get_start_of_month(t - np.timedelta64(24, 'h')) return None