def load_bad_data(project,
site,
data_type,
start_time,
end_time,
archive=None,
path=None,
extension='.bad',
**kwargs):
'''Load data from bad data files, usually those which end in .bad.
This function is a wrapper which calls aurorawatchnet.load_data()
after appending an extension to the file names. The path cannot be
a callable.
'''
if path is None:
ai = ap.get_archive_info(project, site, data_type, archive=archive)
path = ai[1]['path'] + extension
return ap.load_data(project,
site,
data_type,
start_time,
end_time,
archive=ai[0],
path=path,
**kwargs)
def uit_path(t, project, site, data_type, archive, channels):
if uit_password is None:
raise Exception(__name__ + '.uit_password must be set, ' +
'to obtain a password see ' +
'http://flux.phys.uit.no/div/DataAccess.html')
# Expand the path format string with the specific UIT variables,
# including password.
a, d = copy.deepcopy(ap.get_archive_info(project, site, data_type,
archive=archive))
d['uit_password'] = uit_password
fstr = path_fstr % d
return dt64.strftime(t, fstr)
def load_bad_data(project, site, data_type, start_time, end_time,
archive=None, path=None, extension='.bad', **kwargs):
'''Load data from bad data files, usually those which end in .bad.
This function is a wrapper which calls aurorawatchnet.load_data()
after appending an extension to the file names. The path cannot be
a callable.
'''
if path is None:
ai = ap.get_archive_info(project, site, data_type, archive=archive)
path = ai[1]['path'] + extension
return ap.load_data(project, site, data_type, start_time, end_time,
archive=ai[0], path=path, **kwargs)
st = dt64.floor(st, day)
et = dt64.ceil(et, day)
for n in range(len(project_list)):
project = project_list[n]
site = site_list[n]
# Get baseline archive to use for target data
if project in baseline_archive and site in baseline_archive[project]:
bl_archive = baseline_archive[project][site]
else:
bl_archive = 'realtime_baseline'
an, ai = ap.get_archive_info(project, site, 'MagData',
archive=bl_archive)
if 'qdc_fit_duration' not in ai:
logger.error('no qdc_fit_duration found in %s archive for %s/%s',
an, project, site)
continue
if 'realtime_qdc' not in ai:
logger.warning('realtime_qdc option not in %s archive for %s/%s',
an, project, site)
qdc_fit_duration = ai['qdc_fit_duration']
qdc_fit_offset = ai.get('qdc_fit_offset', -qdc_fit_duration/2 - 1.5*day)
qdc_tries = ai.get('qdc_tries', 3)
# Get mag data archive to use for source data
else:
qdc_archive = None
logger.debug('Processing %s/%s' % (project, site))
# Attempt to import missing projects
if project not in ap.projects:
try:
import_module('auroraplot.datasets.' + project.lower())
except:
pass
ax = None
archive, ad = ap.get_archive_info(project, site, 'MagData', \
archive=archive)
qdc_archive, qdc_ad = ap.get_archive_info(project, site, 'MagQDC', \
archive=qdc_archive)
# Tune start/end times to avoid requesting data outside of
# operational period
site_st = ap.get_site_info(project, site, 'start_time')
if site_st is None or site_st < st:
site_st = st
else:
site_st = dt64.floor(site_st, day)
site_et = ap.get_site_info(project, site, 'end_time')
if site_et is None or site_et > et:
site_et = et
else:
if args.single_qdc:
t2 = end_time
else:
t2 = dt64.get_start_of_next_month(t1)
if args.plot:
mag_qdc = ap.magdata.load_qdc(project_uc, site_uc, t1)
if mag_qdc is not None:
lh = mag_qdc.plot(axes=ax)
for h in lh:
h.set_label(dt64.strftime(t1, '%Y-%m-%d'))
ax = plt.gca()
else:
archive, ad = ap.get_archive_info(project_uc, site_uc,
'MagData',
archive=getattr(args,
'archive'))
mag_data = ap.load_data(project_uc, site_uc, 'MagData', t1, t2,
archive=archive,
raise_all=args.raise_all)
if mag_data is not None:
mag_qdc = mag_data.make_qdc(smooth=args.smooth)
qdc_archive, qdc_ad \
= ap.get_archive_info(project_uc, site_uc, 'MagQDC')
if args.single_qdc:
filename = args.single_qdc
else:
filename = dt64.strftime(t1, qdc_ad['path'])
p = os.path.dirname(filename)
archive = ap.parse_archive_selection(args.archive)
else:
archive = {}
for n in range(len(project_list)):
project = project_list[n]
site = site_list[n]
# Get archive to use for data
if project in archive and site in archive[project]:
archive = archive[project][site]
else:
archive = None
dest_an, dest_ai = ap.get_archive_info(project, site, 'MagData',
archive=archive)
src_an, src_ai = ap.get_archive_info(project, site, 'MagData',
archive='original_' + dest_an)
src_path = src_ai['path']
dest_path = dest_ai['path']
print('src_ai ' + src_an)
print(repr(src_ai))
print('dest_ai ' + dest_an)
print(repr(dest_ai))
# Tune start/end times to avoid requesting data outside of
# operational period
site_st = ap.get_site_info(project, site, 'start_time')
if site_st is None or site_st < st:
site_st = st
def make_aurorawatch_plot(project, site, st, et, rolling, exif_tags):
'''
Load data and make the AuroraWatch activity plot. Plots always
cover 24 hours, but may begin at midnight for day plots, or at any
other hour for rolling plots. This function uses the previous 72
hours to help fit the quiet-day curve.
project: name of project
site: name of site
st: start time. For day plots this is the start of the day. For
rolling plots this is the start of the rolling 24 hour period.
et: end time. For day plots this is the start of the following
day. For rolling plots it is the end of the 24 hour period.
rolling: flag to indicate if rolling plot should also be made. It
is not otherwise possible to identify rolling plots which
start at midnight.
'''
# global mag_fstr
global args
# Export to global names for debugging
global mag_data
global mag_qdc
global activity
day = np.timedelta64(24, 'h')
archive, archive_details = ap.get_archive_info(project, site, 'MagData')
# Load the data to plot. For rolling plots load upto midnight so
# that both the rolling plot and the current day plot can be
# generated efficiently.
mag_data = my_load_data(project, site, 'MagData', st, dt64.ceil(et, day))
if mag_data is None or \
not np.any(np.logical_not(np.isnan(mag_data.data))):
# not .np.any(etc) eliminates empty array or array of just nans
logger.info('No magnetic field data')
return
# Load up some data from previous days to and apply a
# least-squares fit to remove baseline drifts. Data from the
# current day is not used. This ensures that results do not change
# over the current day when new data becomes available.
qdc_fit_interval = args.qdc_fit_interval * day
fit_et = dt64.ceil(st, day) # Could be doing a rolling plot
fit_st = fit_et - qdc_fit_interval
fit_data = my_load_data(project, site, 'MagData', fit_st, fit_et)
# Load a QDC.
mag_qdc = ap.magdata.load_qdc(project, site, st, tries=6, realtime=True)
if mag_qdc is None:
logger.info('No QDC')
elif fit_data is None:
# Cannot fit, so assume no errors in QDC
errors = [0.0]
else:
try:
# Fit the QDC to the previous data
qdc_aligned, errors, fi = mag_qdc.align(\
fit_data,
fit=ap.data.Data.minimise_sign_error_fit,
plot_fit=args.plot_fit,
full_output=True)
except Exception as e:
logger.warn('Could not fit QDC')
logger.info(str(e))
errors = [0.0]
else:
# Fitted ok, plot if necessary
if args.plot_fit:
fig = plt.gcf()
fig.set_figwidth(6.4)
fig.set_figheight(4.8)
fig.subplots_adjust(bottom=0.1, top=0.85,
left=0.15, right=0.925)
fit_fstr = mag_fstr[:(mag_fstr.rindex('.'))] + '_fit.png'
mysavefig(fig, dt64.strftime(dt64.ceil(st, day), fit_fstr),
exif_tags)
# Adjust the quiet day curve with the error obtained by fitting to
# previous days.
if mag_qdc is None:
mag_qdc_adj = None
else:
mag_qdc_adj = copy.deepcopy(mag_qdc)
mag_qdc_adj.data -= errors[0]
# Ensure data gaps are marked as such in the plots. Straight lines
# across large gaps look bad!
mag_data = mag_data.mark_missing_data(cadence=2*mag_data.nominal_cadence)
# Do day plot. Trim start time for occasions when making a day
# plot simultaneously with a rolling plot.
st2 = dt64.ceil(st, day)
md_day = mag_data.extract(start_time=st2)
act_ki = activity_plot(md_day, mag_qdc_adj,
dt64.strftime(st2, mag_fstr), exif_tags,
k_index_filename=dt64.strftime(st2, k_fstr))
r = [md_day]
r.extend(act_ki)
if rolling:
# Trim end time
md_rolling = mag_data.extract(end_time=et)
act_ki_rolling = activity_plot(md_rolling, mag_qdc_adj,
rolling_magdata_filename, exif_tags,
k_index_filename=rolling_k_filename)
r.append(md_rolling)
r.extend(act_ki_rolling)
return r
archive = ap.parse_archive_selection(args.archive)
else:
archive = {}
for n in range(len(project_list)):
project = project_list[n]
site = site_list[n]
# Get archive to use for data
if project in archive and site in archive[project]:
archive = archive[project][site]
else:
archive = None
dest_an, dest_ai = ap.get_archive_info(project,
site,
'MagData',
archive=archive)
src_an, src_ai = ap.get_archive_info(project,
site,
'MagData',
archive='original_' + dest_an)
src_path = src_ai['path']
dest_path = dest_ai['path']
print('src_ai ' + src_an)
print(repr(src_ai))
print('dest_ai ' + dest_an)
print(repr(dest_ai))
# Tune start/end times to avoid requesting data outside of
# operational period
baseline_archive = {}
st = dt64.floor(st, day)
et = dt64.ceil(et, day)
for n in range(len(project_list)):
project = project_list[n]
site = site_list[n]
# Get baseline archive to use for target data
if project in baseline_archive and site in baseline_archive[project]:
bl_archive = baseline_archive[project][site]
else:
bl_archive = 'realtime_baseline'
an, ai = ap.get_archive_info(project, site, 'MagData', archive=bl_archive)
if 'qdc_fit_duration' not in ai:
logger.error('no qdc_fit_duration found in %s archive for %s/%s', an,
project, site)
continue
if 'realtime_qdc' not in ai:
logger.warning('realtime_qdc option not in %s archive for %s/%s', an,
project, site)
qdc_fit_duration = ai['qdc_fit_duration']
qdc_fit_offset = ai.get('qdc_fit_offset',
-qdc_fit_duration / 2 - 1.5 * day)
qdc_tries = ai.get('qdc_tries', 3)
qdc_archive = qdc_archive[project][site]
else:
qdc_archive = None
logger.debug('Processing %s/%s' % (project, site))
# Attempt to import missing projects
if project not in ap.projects:
try:
import_module('auroraplot.datasets.' + project.lower())
except:
pass
ax = None
archive, ad = ap.get_archive_info(project, site, 'MagData', \
archive=archive)
qdc_archive, qdc_ad = ap.get_archive_info(project, site, 'MagQDC', \
archive=qdc_archive)
# Tune start/end times to avoid requesting data outside of
# operational period
site_st = ap.get_site_info(project, site, 'start_time')
if site_st is None or site_st < st:
site_st = st
else:
site_st = dt64.floor(site_st, day)
site_et = ap.get_site_info(project, site, 'end_time')
if site_et is None or site_et > et:
site_et = et
else:
def load_qdc(project,
site,
time,
archive=None,
channels=None,
path=None,
tries=1,
realtime=False,
load_function=None,
full_output=False):
'''Load quiet-day curve.
project: name of the project (upper case)
site: site abbreviation (upper case)
time: a time within the quiet-day curve period
The following optional parameters are recognised:
archive: name of the archive. Required if more than one archive is
present and there is not an archive called "default".
channels: data channel(s) to load. All are loaded if not specified.
tries: The number of attempts to load a quiet-day curve. If >1 and
the first attempt is not successful then an attempt will be
made to load the previous QDC.
path: URL or file path, specified as a strftime format specifier.
Alternatively can be a function reference which is passed the
time and returns the filename. If given this overrides the
standard load path.
load_function: Pass responsibility for loading the data to the given
function reference, after validating the input parameters.
'''
data_type = 'MagQDC'
archive, ad = ap.get_archive_info(project, site, data_type,
archive=archive)
if channels is not None:
# Ensure it is a 1D numpy array
channels = np.array(channels).flatten()
for c in channels:
if c not in ad['channels']:
raise ValueError('Unknown channel (%s)' % str(c))
else:
channels = ad['channels']
if path is None:
path = ad['path']
if load_function is None:
load_function = ad.get('load_function', None)
if tries is None:
tries = 1
if load_function:
# Pass responsibility for loading to some other
# function. Parameters have already been checked.
return load_function(project,
site,
data_type,
time,
archive=archive,
channels=channels,
path=path,
tries=tries,
realtime=realtime,
full_output=full_output)
data = []
t = dt64.get_start_of_month(time)
if realtime:
# For realtime use the QDC for the month is (was) not
# available, so use the previous month's QDC
t = dt64.get_start_of_previous_month(t)
# Early in the month the previous motnh's QDC was probably not
# computed, so use the month before that
qdc_rollover_day = ad.get('qdc_rollover_day', 4)
if dt64.get_day_of_month(time) < qdc_rollover_day:
t = dt64.get_start_of_previous_month(t)
for n in range(tries):
try:
if hasattr(path, '__call__'):
# Function: call it with relevant information to get the path
file_name = path(t, project=project, site=site,
data_type=data_type, archive=archive,
channels=channels)
else:
file_name = dt64.strftime(t, path)
logger.info('loading ' + file_name)
r = ad['load_converter'](file_name,
ad,
project=project,
site=site,
data_type=data_type,
start_time=np.timedelta64(0, 'h'),
end_time=np.timedelta64(24, 'h'),
archive=archive,
channels=channels,
path=path)
if r is not None:
r.extract(inplace=True,
channels=channels)
if full_output:
r2 = {'magqdc': r,
'tries': n + 1,
'maxtries': tries}
return r2
else:
return r
finally:
# Go to start of previous month
t = dt64.get_start_of_month(t - np.timedelta64(24, 'h'))
return None
