def temperature_compensation(md, inplace=False, cadence=None, **kwargs):
# Set cadence to reduce noise
cad = np.timedelta64(2, 'm')
if cadence and cadence > cad:
cad = cadence
td = ap.load_data(md.project,
md.site,
'TemperatureData',
md.start_time,
md.end_time,
channels=['Sensor temperature'],
cadence=cad)
# Resample
td = td.interp(md.sample_start_time, md.sample_end_time)
mean_temp = 20
temp_diff = td.data - mean_temp
coeffs = [0, -25e-9, -5e-9]
# coeffs = [0, -2.15532366e-9, -0.49559117e-9]
md_error = np.zeros([1, md.data.shape[1]])
# md_error += coeffs[0] # Offset
# for n in range(1, len(coeffs)):
for n in range(len(coeffs)):
md_error += np.power(temp_diff, n) * coeffs[n]
md_error[np.isnan(md_error)] = 0
if inplace:
r = md
else:
r = copy.deepcopy(md)
logger.debug('temperature compensation error: %f', md_error)
r.data -= md_error
return r
def my_load_data(project, site, data_type, start_time, end_time, **kwargs):
r = ap.load_data(project, site, data_type, start_time, end_time, **kwargs)
if r is not None and args.test_mode:
# Remove any data after 'now' to emulate the correct behaviour
# when using historical data.
r.data[:,r.sample_end_time > now] = np.nan
return r
def load_bad_data(project,
site,
data_type,
start_time,
end_time,
archive=None,
path=None,
extension='.bad',
**kwargs):
'''Load data from bad data files, usually those which end in .bad.
This function is a wrapper which calls aurorawatchnet.load_data()
after appending an extension to the file names. The path cannot be
a callable.
'''
if path is None:
ai = ap.get_archive_info(project, site, data_type, archive=archive)
path = ai[1]['path'] + extension
return ap.load_data(project,
site,
data_type,
start_time,
end_time,
archive=ai[0],
path=path,
**kwargs)
def load_bad_data(project, site, data_type, start_time, end_time,
archive=None, path=None, extension='.bad', **kwargs):
'''Load data from bad data files, usually those which end in .bad.
This function is a wrapper which calls aurorawatchnet.load_data()
after appending an extension to the file names. The path cannot be
a callable.
'''
if path is None:
ai = ap.get_archive_info(project, site, data_type, archive=archive)
path = ai[1]['path'] + extension
return ap.load_data(project, site, data_type, start_time, end_time,
archive=ai[0], path=path, **kwargs)
site_et = ap.get_site_info(project, site, 'end_time')
if site_et is None or site_et > et:
site_et = et
else:
site_et = dt64.ceil(site_et, day)
logger.info('Processing %s/%s %s', project, site, dt64.
fmt_dt64_range(site_st, site_et))
last_data = None
for t1 in dt64.dt64_range(site_st, site_et, day):
try:
t2 = t1 + day
if args.missing_only:
data = ap.load_data(project, site, 'MagData', t1, t2, archive=bl_archive)
if data is not None and np.size(data.data) and np.all(np.isfinite(data.data)):
logger.info('baseline data for %s/%s %s already exists', project, site,
dt64.strftime(t1, '%Y-%m-%d'))
continue
# Calculate dates for data to be used for fitting
md_mean_time = dt64.mean(t1, t2) + qdc_fit_offset
md_st = md_mean_time - qdc_fit_duration/2
md_et = md_st + qdc_fit_duration
if last_data is None or last_data.end_time != md_et - day:
# Load entire data block
md = ap.load_data(project, site, 'MagData', md_st, md_et, archive=md_archive)
else:
# Load the last day of data and concatenate
while t1 < site_et:
t2 = dt64.get_start_of_next_month(t1)
try:
if args.only_missing:
mag_qdc = ap.magdata.load_qdc(project, site, t1)
if mag_qdc is not None and mag_qdc.data.size != 0 and not np.any(np.isnan(mag_qdc.data)):
logger.info('QDC for %s/%s %s already exists', project, site, dt64.strftime(t1, '%Y-%m-%d'))
continue
kwargs = {}
if cadence:
kwargs['cadence'] = cadence
kwargs['aggregate'] = agg_func
mag_data = ap.load_data(project, site, 'MagData', t1, t2,
archive=archive,
raise_all=args.raise_all,
**kwargs)
if mag_data is not None:
if post_cadence:
mag_data.set_cadence(post_cadence,
aggregate=post_agg_func,
inplace=True)
mag_qdc = mag_data.make_qdc(smooth=args.smooth, plot=args.plot)
if args.filename:
filename = args.filename
else:
filename = dt64.strftime(t1, qdc_ad['path'])
if args.plot:
mag_qdc = ap.magdata.load_qdc(project_uc, site_uc, t1)
if mag_qdc is not None:
lh = mag_qdc.plot(axes=ax)
for h in lh:
h.set_label(dt64.strftime(t1, '%Y-%m-%d'))
ax = plt.gca()
else:
archive, ad = ap.get_archive_info(project_uc, site_uc,
'MagData',
archive=getattr(args,
'archive'))
mag_data = ap.load_data(project_uc, site_uc, 'MagData', t1, t2,
archive=archive,
raise_all=args.raise_all)
if mag_data is not None:
mag_qdc = mag_data.make_qdc(smooth=args.smooth)
qdc_archive, qdc_ad \
= ap.get_archive_info(project_uc, site_uc, 'MagQDC')
if args.single_qdc:
filename = args.single_qdc
else:
filename = dt64.strftime(t1, qdc_ad['path'])
p = os.path.dirname(filename)
if not os.path.isdir(p):
os.makedirs(p)
if args.dry_run:
logger.info('Dry run, not saving QDC to ' + filename)
# Requesting the UIT or DTU data from the UIT web site requires a
# password to be set. Try reading the password from a file called
# .uit_password from the user's home directory.
if ('UIT' in project_list or 'DTU' in project_list) \
and ap.datasets.uit.uit_password is None:
raise Exception('UIT password needed but could not be set')
# Load and plot the data for each site.
for n_s in n_s_list:
project, site = n_s.split('/')
kwargs = {}
if project in archives:
kwargs['archive'] = archives[project]
md = ap.load_data(project, site, 'MagData', st, et, **kwargs)
# archive=archives[project])
# Is result is None then no data available, so ignore those
# results.
qdc = None
if (md is not None and
'MagQDC' in ap.get_site_info(project, site, 'data_types')):
md = md.mark_missing_data(cadence=2*md.nominal_cadence)
qdc_info = ap.magdata.load_qdc(project, site, dt64.mean(st, et),
tries=args.tries, full_output=True)
if qdc_info:
qdc = qdc_info['magqdc']
if qdc is not None and len(md.channels) != len(qdc.channels):
qdc = None
k = ap.auroralactivity.KIndex(magdata=md, magqdc=qdc)
k.plot()
post_cadence = None
# Load the data for each site.
mdl = []
for n in range(len(project_list)):
project = project_list[n]
site = site_list[n]
kwargs = {}
if project in archive and site in archive[project]:
kwargs['archive'] = archive[project][site]
if cadence:
kwargs['cadence'] = cadence
kwargs['aggregate'] = agg_func
if ap.is_operational(project, site, st, et):
md = ap.load_data(project, site, data_type, st, et, **kwargs)
else:
logger.info('%s/%s not operational at this time', project, site)
md = None
# If result is None then no data available so ignore those
# results.
if (md is not None and md.data.size and np.any(np.isfinite(md.data))):
md = md.mark_missing_data(cadence=3*md.nominal_cadence)
if post_cadence:
md.set_cadence(post_cadence, aggregate=post_agg_func, inplace=True)
mdl.append(md)
if args.plot_function:
plot_args = dict(plot_func=getattr(plt, args.plot_function))
else:
plot_args = {}
post_cadence = None
# Load the data for each site.
mdl = []
for n in range(len(project_list)):
project = project_list[n]
site = site_list[n]
kwargs = {}
if project in archive and site in archive[project]:
kwargs['archive'] = archive[project][site]
if cadence:
kwargs['cadence'] = cadence
kwargs['aggregate'] = agg_func
if ap.is_operational(project, site, st, et):
md = ap.load_data(project, site, data_type, st, et, **kwargs)
else:
logger.info('%s/%s not operational at this time', project, site)
md = None
# If result is None then no data available so ignore those
# results.
if (md is not None and md.data.size and np.any(np.isfinite(md.data))):
md = md.mark_missing_data(cadence=3 * md.nominal_cadence)
if post_cadence:
md.set_cadence(post_cadence, aggregate=post_agg_func, inplace=True)
mdl.append(md)
if args.plot_function:
plot_args = dict(plot_func=getattr(plt, args.plot_function))
else:
plot_args = {}
site_et = et
else:
site_et = dt64.ceil(site_et, day)
logger.info('Processing %s/%s %s', project, site,
dt64.fmt_dt64_range(site_st, site_et))
last_data = None
for t1 in dt64.dt64_range(site_st, site_et, day):
try:
t2 = t1 + day
if args.missing_only:
data = ap.load_data(project,
site,
'MagData',
t1,
t2,
archive=bl_archive)
if data is not None and np.size(data.data) and np.all(
np.isfinite(data.data)):
logger.info('baseline data for %s/%s %s already exists',
project, site, dt64.strftime(t1, '%Y-%m-%d'))
continue
# Calculate dates for data to be used for fitting
md_mean_time = dt64.mean(t1, t2) + qdc_fit_offset
md_st = md_mean_time - qdc_fit_duration / 2
md_et = md_st + qdc_fit_duration
if last_data is None or last_data.end_time != md_et - day:
# Load entire data block
import numpy as np
import matplotlib as mpl
import matplotlib.pyplot as plt
import auroraplot as ap
from auroraplot.magdata import MagData
import auroraplot.datasets.samnet
logging.basicConfig(level=getattr(logging, 'WARN'))
logger = logging.getLogger(__name__)
st = np.datetime64('2010-01-01T00:00:00+0000')
et = np.datetime64('2010-01-03T00:00:00+0000')
mag_data = ap.load_data('SAMNET', 'LAN1', 'MagData', st, et, archive='5s')
print(mag_data)
st2 = np.datetime64('2013-03-17T00:00:00+0000')
et2 = np.datetime64('2013-03-18T00:00:00+0000')
# Load data from all SAMNET sites
mdl = []
for site in ap.projects['SAMNET']:
try:
tmp = ap.load_data('SAMNET', site, 'MagData', st2, et2, archive='5s')
except:
tmp = None
if tmp is not None and not np.all(np.isnan(tmp.data)) \
and not np.all(tmp.data == 0):
mag_qdc = ap.magdata.load_qdc(project, site, t1)
if mag_qdc is not None and mag_qdc.data.size != 0 and not np.any(
np.isnan(mag_qdc.data)):
logger.info('QDC for %s/%s %s already exists', project,
site, dt64.strftime(t1, '%Y-%m-%d'))
continue
kwargs = {}
if cadence:
kwargs['cadence'] = cadence
kwargs['aggregate'] = agg_func
mag_data = ap.load_data(project,
site,
'MagData',
t1,
t2,
archive=archive,
raise_all=args.raise_all,
**kwargs)
if mag_data is not None:
if post_cadence:
mag_data.set_cadence(post_cadence,
aggregate=post_agg_func,
inplace=True)
mag_qdc = mag_data.make_qdc(smooth=args.smooth, plot=args.plot)
if args.filename:
filename = args.filename
else:
def align(self, ref, fit=None, **fit_kwargs):
day = np.timedelta64(24, 'h')
if isinstance(ref, MagData):
r = copy.deepcopy(ref)
interp_times = ref.get_mean_sample_time()
else:
# ref parameter must contain sample timestamps
assert not fit, 'Cannot fit without reference data'
ta = np.sort(np.array(ref).flatten())
if len(ta) >= 2:
# Guess the nominal cadence
nc = np.median(np.diff(ta))
else:
nc = None
r = MagData(project=self.project,
site=self.site,
channels=copy.copy(self.channels),
start_time=ta[0],
end_time=ta[-1],
sample_start_time=ta,
sample_end_time=copy.copy(ta),
integration_interval=None,
nominal_cadence=nc,
data=None,
units=self.units,
sort=False)
interp_times = ta
# Create array with room for additional entries at start and end
xi = np.zeros([len(self.sample_start_time) + 2], dtype='m8[us]')
xi[1:-1] = self.get_mean_sample_time().astype('m8[us]')
yi = np.zeros([len(self.channels), self.data.shape[1] + 2],
dtype=self.data.dtype)
yi[:,1:-1] = self.data
# Extend so that interpolation can happen correctly near midnight
xi[0] = xi[-2] - day
xi[-1] = xi[1] + day
yi[:, 0] = yi[:, -2]
yi[:, -1] = yi[:, 1]
xo = dt64.get_time_of_day(interp_times).astype('m8[us]')
r.data = scipy.interpolate.interp1d(xi.astype('int64'), yi)\
(xo.astype('int64'))
if fit:
if hasattr(fit, '__call__'):
return fit(r, ref, inplace=True, **fit_kwargs)
elif fit in ('baseline', 'realtime_baseline'):
# QDC must be zero-mean for baseline adjustment,
# correct any offset
for n in range(r.data.shape[0]):
r.data[n] -= ap.nanmean(self.data[n])
for d in np.unique(dt64.get_date(r.sample_start_time)):
bl = ap.load_data(r.project, r.site, 'MagData',
d, d + np.timedelta64(1, 'D'),
channels=r.channels,
archive=fit)
if bl is None or bl.data.size == 0:
raise Exception('No %s MagData for %s/%s %s'
% (fit, self.project, self.site,
str(d)))
r.data[:, d == dt64.get_date(r.sample_start_time)] \
+= bl.data
else:
raise ValueError('Unknown fit method %s' % repr(fit))
return r