コード例 #1
0
def temperature_compensation(md, inplace=False, cadence=None, **kwargs):
    # Set cadence to reduce noise
    cad = np.timedelta64(2, 'm')
    if cadence and cadence > cad:
        cad = cadence
    td = ap.load_data(md.project,
                      md.site,
                      'TemperatureData',
                      md.start_time,
                      md.end_time,
                      channels=['Sensor temperature'],
                      cadence=cad)
    # Resample
    td = td.interp(md.sample_start_time, md.sample_end_time)
    mean_temp = 20
    temp_diff = td.data - mean_temp
    coeffs = [0, -25e-9, -5e-9]
    # coeffs = [0, -2.15532366e-9, -0.49559117e-9]
    md_error = np.zeros([1, md.data.shape[1]])
    # md_error += coeffs[0] # Offset
    # for n in range(1, len(coeffs)):
    for n in range(len(coeffs)):
        md_error += np.power(temp_diff, n) * coeffs[n]

    md_error[np.isnan(md_error)] = 0
    if inplace:
        r = md
    else:
        r = copy.deepcopy(md)
    logger.debug('temperature compensation error: %f', md_error)
    r.data -= md_error
    return r
コード例 #2
0
def my_load_data(project, site, data_type, start_time, end_time, **kwargs):
    r = ap.load_data(project, site, data_type, start_time, end_time, **kwargs)
    if r is not None and args.test_mode:
        # Remove any data after 'now' to emulate the correct behaviour
        # when using historical data.
        r.data[:,r.sample_end_time > now] = np.nan
    return r
コード例 #3
0
def load_bad_data(project,
                  site,
                  data_type,
                  start_time,
                  end_time,
                  archive=None,
                  path=None,
                  extension='.bad',
                  **kwargs):
    '''Load data from bad data files, usually those which end in .bad.

    This function is a wrapper which calls aurorawatchnet.load_data()
    after appending an extension to the file names. The path cannot be
    a callable.
    '''
    if path is None:
        ai = ap.get_archive_info(project, site, data_type, archive=archive)
        path = ai[1]['path'] + extension

    return ap.load_data(project,
                        site,
                        data_type,
                        start_time,
                        end_time,
                        archive=ai[0],
                        path=path,
                        **kwargs)
コード例 #4
0
def load_bad_data(project, site, data_type, start_time, end_time,
                  archive=None, path=None, extension='.bad', **kwargs):
    '''Load data from bad data files, usually those which end in .bad.

    This function is a wrapper which calls aurorawatchnet.load_data()
    after appending an extension to the file names. The path cannot be
    a callable.
    '''
    if path is None:
        ai = ap.get_archive_info(project, site, data_type, archive=archive)
        path = ai[1]['path'] + extension

    return ap.load_data(project, site, data_type, start_time, end_time,
                        archive=ai[0], path=path, **kwargs)
コード例 #5
0
    site_et = ap.get_site_info(project, site, 'end_time')        
    if site_et is None or site_et > et:
        site_et = et
    else:
        site_et = dt64.ceil(site_et, day)

    logger.info('Processing %s/%s %s', project, site, dt64.
                fmt_dt64_range(site_st, site_et))

    last_data = None
    for t1 in dt64.dt64_range(site_st, site_et, day):
        try:
            t2 = t1 + day

            if args.missing_only:
                data = ap.load_data(project, site, 'MagData', t1, t2, archive=bl_archive)
                if data is not None and np.size(data.data) and np.all(np.isfinite(data.data)):
                    logger.info('baseline data for %s/%s %s already exists', project, site,
                                dt64.strftime(t1, '%Y-%m-%d'))
                    continue

            # Calculate dates for data to be used for fitting
            md_mean_time = dt64.mean(t1, t2) + qdc_fit_offset
            md_st = md_mean_time - qdc_fit_duration/2
            md_et = md_st + qdc_fit_duration

            if last_data is None or last_data.end_time != md_et - day:
                # Load entire data block
                md = ap.load_data(project, site, 'MagData', md_st, md_et, archive=md_archive)
            else:
                # Load the last day of data and concatenate
コード例 #6
0
ファイル: make_qdc.py プロジェクト: stevemarple/auroraplot
    while t1 < site_et:
        t2 = dt64.get_start_of_next_month(t1)
        try:
            if args.only_missing:
                mag_qdc = ap.magdata.load_qdc(project, site, t1)
                if mag_qdc is not None and mag_qdc.data.size != 0 and not np.any(np.isnan(mag_qdc.data)):
                    logger.info('QDC for %s/%s %s already exists', project, site, dt64.strftime(t1, '%Y-%m-%d'))
                    continue

            kwargs = {}
            if cadence:
                kwargs['cadence'] = cadence
                kwargs['aggregate'] = agg_func

            mag_data = ap.load_data(project, site, 'MagData', t1, t2,
                                    archive=archive,
                                    raise_all=args.raise_all,
                                    **kwargs)
            if mag_data is not None:

                if post_cadence:
                    mag_data.set_cadence(post_cadence, 
                                         aggregate=post_agg_func,
                                         inplace=True)

                mag_qdc = mag_data.make_qdc(smooth=args.smooth, plot=args.plot)

                if args.filename:
                    filename = args.filename
                else:
                    filename = dt64.strftime(t1, qdc_ad['path'])
コード例 #7
0
        if args.plot:
            mag_qdc = ap.magdata.load_qdc(project_uc, site_uc, t1)
            if mag_qdc is not None:
                lh = mag_qdc.plot(axes=ax)
                for h in lh:
                    h.set_label(dt64.strftime(t1, '%Y-%m-%d'))
                ax = plt.gca()

        else:
            archive, ad = ap.get_archive_info(project_uc, site_uc, 
                                              'MagData', 
                                              archive=getattr(args, 
                                                              'archive'))

            mag_data = ap.load_data(project_uc, site_uc, 'MagData', t1, t2,
                                    archive=archive,
                                    raise_all=args.raise_all)
            if mag_data is not None:
                mag_qdc = mag_data.make_qdc(smooth=args.smooth)
                qdc_archive, qdc_ad \
                    = ap.get_archive_info(project_uc, site_uc, 'MagQDC')

                if args.single_qdc:
                    filename = args.single_qdc
                else:
                    filename = dt64.strftime(t1, qdc_ad['path'])
                p = os.path.dirname(filename)
                if not os.path.isdir(p):
                    os.makedirs(p)
                if args.dry_run:
                    logger.info('Dry run, not saving QDC to ' + filename)
コード例 #8
0
# Requesting the UIT or DTU data from the UIT web site requires a
# password to be set. Try reading the password from a file called
# .uit_password from the user's home directory.
if ('UIT' in project_list or 'DTU' in project_list) \
        and ap.datasets.uit.uit_password is None:
    raise Exception('UIT password needed but could not be set')


# Load and plot the data for each site. 
for n_s in n_s_list:
    project, site = n_s.split('/')
    kwargs = {}
    if project in archives:
        kwargs['archive'] = archives[project]
    md = ap.load_data(project, site, 'MagData', st, et, **kwargs)
                      # archive=archives[project])
    # Is result is None then no data available, so ignore those
    # results.
    qdc = None
    if (md is not None and
        'MagQDC' in ap.get_site_info(project, site, 'data_types')):
        md = md.mark_missing_data(cadence=2*md.nominal_cadence)
        qdc_info = ap.magdata.load_qdc(project, site, dt64.mean(st, et),
                                       tries=args.tries, full_output=True)
        if qdc_info:
            qdc = qdc_info['magqdc']
        if qdc is not None and len(md.channels) != len(qdc.channels):
            qdc = None
    k = ap.auroralactivity.KIndex(magdata=md, magqdc=qdc)
    k.plot()
コード例 #9
0
    post_cadence = None

# Load the data for each site. 
mdl = []
for n in range(len(project_list)):
    project = project_list[n]
    site = site_list[n]
    kwargs = {}
    if project in archive and site in archive[project]:
        kwargs['archive'] = archive[project][site]
    if cadence:
        kwargs['cadence'] = cadence
        kwargs['aggregate'] = agg_func

    if ap.is_operational(project, site, st, et):
        md = ap.load_data(project, site, data_type, st, et, **kwargs)
    else:
        logger.info('%s/%s not operational at this time', project, site)
        md = None
    # If result is None then no data available so ignore those
    # results.
    if (md is not None and md.data.size and np.any(np.isfinite(md.data))):
        md = md.mark_missing_data(cadence=3*md.nominal_cadence)
        if post_cadence:
            md.set_cadence(post_cadence, aggregate=post_agg_func, inplace=True)
        mdl.append(md)

if args.plot_function:
    plot_args = dict(plot_func=getattr(plt, args.plot_function))
else:
    plot_args = {}
コード例 #10
0
    post_cadence = None

# Load the data for each site.
mdl = []
for n in range(len(project_list)):
    project = project_list[n]
    site = site_list[n]
    kwargs = {}
    if project in archive and site in archive[project]:
        kwargs['archive'] = archive[project][site]
    if cadence:
        kwargs['cadence'] = cadence
        kwargs['aggregate'] = agg_func

    if ap.is_operational(project, site, st, et):
        md = ap.load_data(project, site, data_type, st, et, **kwargs)
    else:
        logger.info('%s/%s not operational at this time', project, site)
        md = None
    # If result is None then no data available so ignore those
    # results.
    if (md is not None and md.data.size and np.any(np.isfinite(md.data))):
        md = md.mark_missing_data(cadence=3 * md.nominal_cadence)
        if post_cadence:
            md.set_cadence(post_cadence, aggregate=post_agg_func, inplace=True)
        mdl.append(md)

if args.plot_function:
    plot_args = dict(plot_func=getattr(plt, args.plot_function))
else:
    plot_args = {}
コード例 #11
0
        site_et = et
    else:
        site_et = dt64.ceil(site_et, day)

    logger.info('Processing %s/%s %s', project, site,
                dt64.fmt_dt64_range(site_st, site_et))

    last_data = None
    for t1 in dt64.dt64_range(site_st, site_et, day):
        try:
            t2 = t1 + day

            if args.missing_only:
                data = ap.load_data(project,
                                    site,
                                    'MagData',
                                    t1,
                                    t2,
                                    archive=bl_archive)
                if data is not None and np.size(data.data) and np.all(
                        np.isfinite(data.data)):
                    logger.info('baseline data for %s/%s %s already exists',
                                project, site, dt64.strftime(t1, '%Y-%m-%d'))
                    continue

            # Calculate dates for data to be used for fitting
            md_mean_time = dt64.mean(t1, t2) + qdc_fit_offset
            md_st = md_mean_time - qdc_fit_duration / 2
            md_et = md_st + qdc_fit_duration

            if last_data is None or last_data.end_time != md_et - day:
                # Load entire data block
コード例 #12
0
import numpy as np
import matplotlib as mpl
import matplotlib.pyplot as plt

import auroraplot as ap
from auroraplot.magdata import MagData

import auroraplot.datasets.samnet

logging.basicConfig(level=getattr(logging, 'WARN'))
logger = logging.getLogger(__name__)

st = np.datetime64('2010-01-01T00:00:00+0000')
et = np.datetime64('2010-01-03T00:00:00+0000')

mag_data = ap.load_data('SAMNET', 'LAN1', 'MagData', st, et, archive='5s')
print(mag_data)

st2 = np.datetime64('2013-03-17T00:00:00+0000')
et2 = np.datetime64('2013-03-18T00:00:00+0000')

# Load data from all SAMNET sites
mdl = []
for site in ap.projects['SAMNET']:
    try:
        tmp = ap.load_data('SAMNET', site, 'MagData', st2, et2, archive='5s')
    except:
        tmp = None

    if tmp is not None and not np.all(np.isnan(tmp.data)) \
            and not np.all(tmp.data == 0):
コード例 #13
0
                mag_qdc = ap.magdata.load_qdc(project, site, t1)
                if mag_qdc is not None and mag_qdc.data.size != 0 and not np.any(
                        np.isnan(mag_qdc.data)):
                    logger.info('QDC for %s/%s %s already exists', project,
                                site, dt64.strftime(t1, '%Y-%m-%d'))
                    continue

            kwargs = {}
            if cadence:
                kwargs['cadence'] = cadence
                kwargs['aggregate'] = agg_func

            mag_data = ap.load_data(project,
                                    site,
                                    'MagData',
                                    t1,
                                    t2,
                                    archive=archive,
                                    raise_all=args.raise_all,
                                    **kwargs)
            if mag_data is not None:

                if post_cadence:
                    mag_data.set_cadence(post_cadence,
                                         aggregate=post_agg_func,
                                         inplace=True)

                mag_qdc = mag_data.make_qdc(smooth=args.smooth, plot=args.plot)

                if args.filename:
                    filename = args.filename
                else:
コード例 #14
0
    def align(self, ref, fit=None, **fit_kwargs):
        day = np.timedelta64(24, 'h')
        if isinstance(ref, MagData):
            r = copy.deepcopy(ref)
            interp_times = ref.get_mean_sample_time()
        else:
            # ref parameter must contain sample timestamps
            assert not fit, 'Cannot fit without reference data'
            ta = np.sort(np.array(ref).flatten())
            if len(ta) >= 2:
                # Guess the nominal cadence
                nc = np.median(np.diff(ta))
            else:
                nc = None
            r = MagData(project=self.project,
                        site=self.site,
                        channels=copy.copy(self.channels),
                        start_time=ta[0],
                        end_time=ta[-1],
                        sample_start_time=ta,
                        sample_end_time=copy.copy(ta),
                        integration_interval=None,
                        nominal_cadence=nc,
                        data=None,
                        units=self.units,
                        sort=False)
            interp_times = ta

        # Create array with room for additional entries at start and end
        xi = np.zeros([len(self.sample_start_time) + 2], dtype='m8[us]')
        xi[1:-1] = self.get_mean_sample_time().astype('m8[us]')
        yi = np.zeros([len(self.channels), self.data.shape[1] + 2],
                      dtype=self.data.dtype)
        yi[:,1:-1] = self.data
        # Extend so that interpolation can happen correctly near midnight
        xi[0] = xi[-2] - day
        xi[-1] = xi[1] + day
        yi[:, 0] = yi[:, -2]
        yi[:, -1] = yi[:, 1]

        xo = dt64.get_time_of_day(interp_times).astype('m8[us]')
        r.data = scipy.interpolate.interp1d(xi.astype('int64'), yi)\
            (xo.astype('int64'))

        if fit:
            if hasattr(fit, '__call__'):
                return fit(r, ref, inplace=True, **fit_kwargs)
            elif fit in ('baseline', 'realtime_baseline'):
                # QDC must be zero-mean for baseline adjustment,
                # correct any offset
                for n in range(r.data.shape[0]):
                    r.data[n] -= ap.nanmean(self.data[n])
                for d in np.unique(dt64.get_date(r.sample_start_time)):
                    bl = ap.load_data(r.project, r.site, 'MagData',
                                      d, d + np.timedelta64(1, 'D'),
                                      channels=r.channels,
                                      archive=fit)
                    if bl is None or bl.data.size == 0:
                        raise Exception('No %s MagData for %s/%s %s' 
                                        % (fit, self.project, self.site,
                                           str(d)))
                    r.data[:, d == dt64.get_date(r.sample_start_time)] \
                        += bl.data
            else:
                raise ValueError('Unknown fit method %s' % repr(fit))
        return r