def compute_mag_qdc_t(st):
'''Compute QDC time.
For the 4th or later in the month load the previous
month, otherwise go back two months. This gives a few days for
data to be transferred and QDCs to be made and checked.'''
qdc_t = dt64.get_start_of_previous_month(st)
if dt64.get_day_of_month(st) < 4:
qdc_t = dt64.get_start_of_previous_month(qdc_t)
return qdc_t
def make_aurorawatch_plot(project, site, st, et, rolling, exif_tags):
'''
Load data and make the AuroraWatch activity plot. Plots always
cover 24 hours, but may begin at midnight for day plots, or at any
other hour for rolling plots. This function uses the previous 72
hours to help fit the quiet-day curve.
project: name of project
site: name of site
st: start time. For day plots this is the start of the day. For
rolling plots this is the start of the rolling 24 hour period.
et: end time. For day plots this is the start of the following
day. For rolling plots it is the end of the 24 hour period.
rolling: flag to indicate if rolling plot should also be made. It
is not otherwise possible to identify rolling plots which
start at midnight.
'''
# global mag_fstr
global args
# Export to global names for debugging
global mag_data
global mag_qdc
global activity
day = np.timedelta64(24, 'h')
archive, archive_details = ap.get_archive_info(project, site, 'MagData')
# Load the data to plot. For rolling plots load upto midnight so
# that both the rolling plot and the current day plot can be
# generated efficiently.
mag_data = my_load_data(project, site, 'MagData', st, dt64.ceil(et, day))
if mag_data is None or \
not np.any(np.logical_not(np.isnan(mag_data.data))):
# not .np.any(etc) eliminates empty array or array of just nans
logger.info('No magnetic field data')
return
# Load up some data from previous days to and apply a
# least-squares fit to remove baseline drifts. Data from the
# current day is not used. This ensures that results do not change
# over the current day when new data becomes available.
qdc_fit_interval = args.qdc_fit_interval * day
fit_et = dt64.ceil(st, day) # Could be doing a rolling plot
fit_st = fit_et - qdc_fit_interval
fit_data = my_load_data(project, site, 'MagData', fit_st, fit_et)
# Load a QDC. For the 4th or later in the month load the previous
# month, otherwise go back two months. This gives a few days for
# data to be transferred, and QDCs to be made and checked.
qdc_t = dt64.get_start_of_previous_month(st)
if dt64.get_day_of_month(st) < 4:
qdc_t = dt64.get_start_of_previous_month(qdc_t)
mag_qdc = ap.magdata.load_qdc(project, site, qdc_t, tries=6)
if mag_qdc is None:
logger.info('No QDC')
elif fit_data is None:
# Cannot fit, so assume no errors in QDC
errors = [0.0]
else:
try:
# Fit the QDC to the previous data
qdc_aligned, errors, fi = mag_qdc.align(\
fit_data,
fit=ap.data.Data.minimise_sign_error_fit,
plot_fit=args.plot_fit,
full_output=True)
except Exception as e:
logger.warn('Could not fit QDC')
logger.info(str(e))
errors = [0.0]
else:
# Fitted ok, plot if necessary
if args.plot_fit:
fig = plt.gcf()
fig.set_figwidth(6.4)
fig.set_figheight(4.8)
fig.subplots_adjust(bottom=0.1, top=0.85,
left=0.15, right=0.925)
fit_fstr = mag_fstr[:(mag_fstr.rindex('.'))] + '_fit.png'
mysavefig(fig, dt64.strftime(dt64.ceil(st, day), fit_fstr),
exif_tags)
# Adjust the quiet day curve with the error obtained by fitting to
# previous days.
if mag_qdc is None:
mag_qdc_adj = None
else:
mag_qdc_adj = copy.deepcopy(mag_qdc)
mag_qdc_adj.data -= errors[0]
# Ensure data gaps are marked as such in the plots. Straight lines
# across large gaps look bad!
mag_data = mag_data.mark_missing_data(cadence=2*mag_data.nominal_cadence)
# Do day plot. Trim start time for occasions when making a day
# plot simultaneously with a rolling plot.
st2 = dt64.ceil(st, day)
md_day = mag_data.extract(start_time=st2)
act_ki = activity_plot(md_day, mag_qdc_adj,
dt64.strftime(st2, mag_fstr), exif_tags,
k_index_filename=dt64.strftime(st2, k_fstr))
r = [md_day]
r.extend(act_ki)
if rolling:
# Trim end time
md_rolling = mag_data.extract(end_time=et)
act_ki_rolling = activity_plot(md_rolling, mag_qdc_adj,
rolling_magdata_filename, exif_tags,
k_index_filename=rolling_k_filename)
r.append(md_rolling)
r.extend(act_ki_rolling)
return r
args = parser.parse_args()
if __name__ == '__main__':
logging.basicConfig(level=getattr(logging, args.log_level.upper()),
format=args.log_format)
# Use a consistent value for current time
day = np.timedelta64(1, 'D').astype('m8[us]')
now = np.datetime64('now', 'us')
today = dt64.floor(now, day)
yesterday = today - day
tomorrow = today + day
if args.start_time is None:
start_time = dt64.get_start_of_previous_month(today)
elif args.start_time == 'today':
start_time = today
elif args.start_time == 'yesterday':
start_time = yesterday
else:
start_time = dt64.floor(np.datetime64(args.start_time), day)
if args.end_time is None:
end_time = start_time + day
elif args.end_time == 'today':
end_time = today
elif args.end_time == 'yesterday':
end_time = yesterday
elif args.end_time == 'tomorrow':
end_time = tomorrow
def load_qdc(project,
site,
time,
archive=None,
channels=None,
path=None,
tries=1,
realtime=False,
load_function=None,
full_output=False):
'''Load quiet-day curve.
project: name of the project (upper case)
site: site abbreviation (upper case)
time: a time within the quiet-day curve period
The following optional parameters are recognised:
archive: name of the archive. Required if more than one archive is
present and there is not an archive called "default".
channels: data channel(s) to load. All are loaded if not specified.
tries: The number of attempts to load a quiet-day curve. If >1 and
the first attempt is not successful then an attempt will be
made to load the previous QDC.
path: URL or file path, specified as a strftime format specifier.
Alternatively can be a function reference which is passed the
time and returns the filename. If given this overrides the
standard load path.
load_function: Pass responsibility for loading the data to the given
function reference, after validating the input parameters.
'''
data_type = 'MagQDC'
archive, ad = ap.get_archive_info(project, site, data_type,
archive=archive)
if channels is not None:
# Ensure it is a 1D numpy array
channels = np.array(channels).flatten()
for c in channels:
if c not in ad['channels']:
raise ValueError('Unknown channel (%s)' % str(c))
else:
channels = ad['channels']
if path is None:
path = ad['path']
if load_function is None:
load_function = ad.get('load_function', None)
if tries is None:
tries = 1
if load_function:
# Pass responsibility for loading to some other
# function. Parameters have already been checked.
return load_function(project,
site,
data_type,
time,
archive=archive,
channels=channels,
path=path,
tries=tries,
realtime=realtime,
full_output=full_output)
data = []
t = dt64.get_start_of_month(time)
if realtime:
# For realtime use the QDC for the month is (was) not
# available, so use the previous month's QDC
t = dt64.get_start_of_previous_month(t)
# Early in the month the previous motnh's QDC was probably not
# computed, so use the month before that
qdc_rollover_day = ad.get('qdc_rollover_day', 4)
if dt64.get_day_of_month(time) < qdc_rollover_day:
t = dt64.get_start_of_previous_month(t)
for n in range(tries):
try:
if hasattr(path, '__call__'):
# Function: call it with relevant information to get the path
file_name = path(t, project=project, site=site,
data_type=data_type, archive=archive,
channels=channels)
else:
file_name = dt64.strftime(t, path)
logger.info('loading ' + file_name)
r = ad['load_converter'](file_name,
ad,
project=project,
site=site,
data_type=data_type,
start_time=np.timedelta64(0, 'h'),
end_time=np.timedelta64(24, 'h'),
archive=archive,
channels=channels,
path=path)
if r is not None:
r.extract(inplace=True,
channels=channels)
if full_output:
r2 = {'magqdc': r,
'tries': n + 1,
'maxtries': tries}
return r2
else:
return r
finally:
# Go to start of previous month
t = dt64.get_start_of_month(t - np.timedelta64(24, 'h'))
return None
