Ejemplo n.º 1
0
def load(fnames,
         tag=None,
         sat_id=None,
         sim_multi_file_right=False,
         sim_multi_file_left=False,
         root_date=None,
         file_date_range=None,
         malformed_index=False,
         mangle_file_dates=False):
    """ Loads the test files

    Parameters
    ----------
    fnames : list
        List of filenames
    tag : str or NoneType
        Instrument tag (accepts '')
    sat_id : str or NoneType
        Instrument satellite ID (accepts '' or a number (i.e., '10'), which
        specifies the number of data points to include in the test instrument)
    sim_multi_file_right : boolean
        Adjusts date range to be 12 hours in the future or twelve hours beyond
        root_date (default=False)
    sim_multi_file_left : boolean
        Adjusts date range to be 12 hours in the past or twelve hours before
        root_date (default=False)
    root_date : NoneType
        Optional central date, uses _test_dates if not specified.
        (default=None)
    file_date_range : pds.date_range or NoneType
        Range of dates for files or None, if this optional arguement is not
        used. Shift actually performed by the init function.
        (default=None)
    malformed_index : boolean
        If True, time index will be non-unique and non-monotonic.
    mangle_file_dates : bool
        If True, the loaded file list time index is shifted by 5-minutes.
        This shift is actually performed by the init function.

    Returns
    -------
    data : (pds.DataFrame)
        Testing data
    meta : (pysat.Meta)
        Metadataxs

    """

    # create an artifical satellite data set
    iperiod = mm_test.define_period()
    drange = mm_test.define_range()
    uts, index, date = mm_test.generate_times(fnames, sat_id, freq='1S')

    # Specify the date tag locally and determine the desired date range
    pds_offset = pds.DateOffset(hours=12)
    if sim_multi_file_right:
        root_date = root_date or _test_dates[''][''] + pds_offset
    elif sim_multi_file_left:
        root_date = root_date or _test_dates[''][''] - pds_offset
    else:
        root_date = root_date or _test_dates['']['']

    data = pysat.DataFrame(uts, columns=['uts'])

    # need to create simple orbits here. Have start of first orbit default
    # to 1 Jan 2009, 00:00 UT. 14.84 orbits per day
    time_delta = date - root_date
    data['mlt'] = mm_test.generate_fake_data(time_delta.total_seconds(),
                                             uts,
                                             period=iperiod['lt'],
                                             data_range=drange['lt'])

    # do slt, 20 second offset from mlt
    data['slt'] = mm_test.generate_fake_data(time_delta.total_seconds() + 20,
                                             uts,
                                             period=iperiod['lt'],
                                             data_range=drange['lt'])

    # create a fake longitude, resets every 6240 seconds
    # sat moves at 360/5820 deg/s, Earth rotates at 360/86400, takes extra time
    # to go around full longitude
    data['longitude'] = mm_test.generate_fake_data(time_delta.total_seconds(),
                                                   uts,
                                                   period=iperiod['lon'],
                                                   data_range=drange['lon'])

    # create latitude area for testing polar orbits
    angle = mm_test.generate_fake_data(time_delta.total_seconds(),
                                       uts,
                                       period=iperiod['angle'],
                                       data_range=drange['angle'])
    data['latitude'] = 90.0 * np.cos(angle)

    # fake orbit number
    fake_delta = date - (_test_dates[''][''] - pds.DateOffset(years=1))
    data['orbit_num'] = mm_test.generate_fake_data(fake_delta.total_seconds(),
                                                   uts,
                                                   period=iperiod['lt'],
                                                   cyclic=False)

    # create some fake data to support testing of averaging routines
    mlt_int = data['mlt'].astype(int)
    long_int = (data['longitude'] / 15.0).astype(int)
    if tag == 'ascend':
        data['dummy1'] = [i for i in range(len(data['mlt']))]
    elif tag == 'descend':
        data['dummy1'] = [-i for i in range(len(data['mlt']))]
    elif tag == 'plus10':
        data['dummy1'] = [i + 10 for i in range(len(data['mlt']))]
    elif tag == 'fives':
        data['dummy1'] = [5 for i in range(len(data['mlt']))]
    elif tag == 'mlt_offset':
        data['dummy1'] = mlt_int + 5
    else:
        data['dummy1'] = mlt_int
    data['dummy2'] = long_int
    data['dummy3'] = mlt_int + long_int * 1000.0
    data['dummy4'] = uts
    data['string_dummy'] = ['test'] * len(data)
    data['unicode_dummy'] = [u'test'] * len(data)
    data['int8_dummy'] = np.ones(len(data), dtype=np.int8)
    data['int16_dummy'] = np.ones(len(data), dtype=np.int16)
    data['int32_dummy'] = np.ones(len(data), dtype=np.int32)
    data['int64_dummy'] = np.ones(len(data), dtype=np.int64)

    if malformed_index:
        index = index.tolist()
        # nonmonotonic
        index[0:3], index[3:6] = index[3:6], index[0:3]
        # non unique
        index[6:9] = [index[6]] * 3

    data.index = index
    data.index.name = 'Epoch'
    return data, meta.copy()
Ejemplo n.º 2
0
def load(fnames,
         tag=None,
         inst_id=None,
         malformed_index=False,
         num_samples=None,
         test_load_kwarg=None):
    """ Loads the test files

    Parameters
    ----------
    fnames : list
        List of filenames
    tag : str or NoneType
        Instrument tag (accepts '')
    inst_id : str or NoneType
        Instrument satellite ID (accepts '')
    malformed_index : bool
        If True, the time index will be non-unique and non-monotonic.
        (default=False)
    num_samples : int
        Number of samples
    test_load_kwarg : any or NoneType
        Testing keyword (default=None)

    Returns
    -------
    data : pds.DataFrame
        Testing data
    meta : pysat.Meta
        Testing metadata

    """

    # Support keyword testing
    logger.info(''.join(('test_load_kwarg = ', str(test_load_kwarg))))

    # create an artifical satellite data set
    iperiod = mm_test.define_period()
    drange = mm_test.define_range()
    if num_samples is None:
        # Default to 1 day at a frequency of 100S
        num_samples = 864

    # Using 100s frequency for compatibility with seasonal analysis unit tests
    uts, index, dates = mm_test.generate_times(fnames,
                                               num_samples,
                                               freq='100S')
    # seed DataFrame with UT array
    data = pds.DataFrame(np.mod(uts, 86400.), columns=['uts'])

    # need to create simple orbits here. Have start of first orbit
    # at 2009,1, 0 UT. 14.84 orbits per day
    # figure out how far in time from the root start
    # use that info to create a signal that is continuous from that start
    # going to presume there are 5820 seconds per orbit (97 minute period)
    time_delta = dates[0] - dt.datetime(2009, 1, 1)
    # mlt runs 0-24 each orbit.
    data['mlt'] = mm_test.generate_fake_data(time_delta.total_seconds(),
                                             uts,
                                             period=iperiod['lt'],
                                             data_range=drange['lt'])
    # do slt, 20 second offset from mlt
    data['slt'] = mm_test.generate_fake_data(time_delta.total_seconds() + 20,
                                             uts,
                                             period=iperiod['lt'],
                                             data_range=drange['lt'])
    # create a fake longitude, resets every 6240 seconds
    # sat moves at 360/5820 deg/s, Earth rotates at 360/86400, takes extra time
    # to go around full longitude
    data['longitude'] = mm_test.generate_fake_data(time_delta.total_seconds(),
                                                   uts,
                                                   period=iperiod['lon'],
                                                   data_range=drange['lon'])
    # create latitude signal for testing polar orbits
    angle = mm_test.generate_fake_data(time_delta.total_seconds(),
                                       uts,
                                       period=iperiod['angle'],
                                       data_range=drange['angle'])
    data['latitude'] = 90.0 * np.cos(angle)

    # create constant altitude at 400 km
    alt0 = 400.0
    data['altitude'] = alt0 * np.ones(data['latitude'].shape)

    if malformed_index:
        index = index.tolist()
        # nonmonotonic
        index[0:3], index[3:6] = index[3:6], index[0:3]
        # non unique
        index[6:9] = [index[6]] * 3

    data.index = index
    data.index.name = 'epoch'
    # higher rate time signal (for scalar >= 2)
    # this time signal used for 2D profiles associated with each time in main
    # DataFrame
    num_profiles = 50 if num_samples >= 50 else num_samples
    end_date = dates[0] + dt.timedelta(seconds=2 * num_profiles - 1)
    high_rate_template = pds.date_range(dates[0], end_date, freq='2S')

    # create a few simulated profiles
    # DataFrame at each time with mixed variables
    profiles = []
    # DataFrame at each time with numeric variables only
    alt_profiles = []
    # Serie at each time, numeric data only
    series_profiles = []
    # frame indexed by date times
    frame = pds.DataFrame(
        {
            'density': data.loc[data.index[0:num_profiles],
                                'mlt'].values.copy(),
            'dummy_str': ['test'] * num_profiles,
            'dummy_ustr': [u'test'] * num_profiles
        },
        index=data.index[0:num_profiles],
        columns=['density', 'dummy_str', 'dummy_ustr'])
    # frame indexed by float
    dd = np.arange(num_profiles) * 1.2
    ff = np.arange(num_profiles) / num_profiles
    ii = np.arange(num_profiles) * 0.5
    frame_alt = pds.DataFrame({
        'density': dd,
        'fraction': ff
    },
                              index=ii,
                              columns=['density', 'fraction'])
    # series version of storage
    series_alt = pds.Series(dd, index=ii, name='series_profiles')

    for time in data.index:
        frame.index = high_rate_template + (time - data.index[0])
        profiles.append(frame)
        alt_profiles.append(frame_alt)
        series_profiles.append(series_alt)
    # store multiple data types into main frame
    data['profiles'] = pds.Series(profiles, index=data.index)
    data['alt_profiles'] = pds.Series(alt_profiles, index=data.index)
    data['series_profiles'] = pds.Series(series_profiles, index=data.index)

    # create very limited metadata
    meta = pysat.Meta()
    meta['uts'] = {'units': 's', 'long_name': 'Universal Time'}
    meta['mlt'] = {'units': 'hours', 'long_name': 'Magnetic Local Time'}
    meta['slt'] = {'units': 'hours', 'long_name': 'Solar Local Time'}
    meta['longitude'] = {'units': 'degrees', 'long_name': 'Longitude'}
    meta['latitude'] = {'units': 'degrees', 'long_name': 'Latitude'}
    meta['altitude'] = {'units': 'km', 'long_name': 'Altitude'}
    series_profile_meta = pysat.Meta()
    series_profile_meta['series_profiles'] = {'long_name': 'series'}
    meta['series_profiles'] = {
        'meta': series_profile_meta,
        'long_name': 'series'
    }
    profile_meta = pysat.Meta()
    profile_meta['density'] = {'long_name': 'profiles'}
    profile_meta['dummy_str'] = {'long_name': 'profiles'}
    profile_meta['dummy_ustr'] = {'long_name': 'profiles'}
    meta['profiles'] = {'meta': profile_meta, 'long_name': 'profiles'}
    alt_profile_meta = pysat.Meta()
    alt_profile_meta['density'] = {'long_name': 'profiles'}
    alt_profile_meta['fraction'] = {'long_name': 'profiles'}
    meta['alt_profiles'] = {'meta': alt_profile_meta, 'long_name': 'profiles'}

    return data, meta
Ejemplo n.º 3
0
def load(fnames,
         tag=None,
         inst_id=None,
         malformed_index=False,
         num_samples=None,
         test_load_kwarg=None):
    """ Loads the test files

    Parameters
    ----------
    fnames : list
        List of filenames
    tag : str or NoneType
        Instrument tag (accepts '')
    inst_id : str or NoneType
        Instrument satellite ID (accepts '')
    malformed_index : bool False
        If True, the time index will be non-unique and non-monotonic.
    num_samples : int
        Number of samples
    test_load_kwarg : any or NoneType
        Testing keyword (default=None)

    Returns
    -------
    data : xr.Dataset
        Testing data
    meta : pysat.Meta
        Testing metadata

    """

    # Support keyword testing
    logger.info(''.join(('test_load_kwarg = ', str(test_load_kwarg))))

    # create an artifical satellite data set
    iperiod = mm_test.define_period()
    drange = mm_test.define_range()

    if num_samples is None:
        # Default to 1 day at a frequency of 100S
        num_samples = 864
    # Using 100s frequency for compatibility with seasonal analysis unit tests
    uts, index, dates = mm_test.generate_times(fnames,
                                               num_samples,
                                               freq='100S')

    if malformed_index:
        index = index.tolist()
        # nonmonotonic
        index[0:3], index[3:6] = index[3:6], index[0:3]
        # non unique
        index[6:9] = [index[6]] * 3
    data = xr.Dataset({'uts': ((epoch_name), index)},
                      coords={epoch_name: index})

    # need to create simple orbits here. Have start of first orbit
    # at 2009,1, 0 UT. 14.84 orbits per day
    # figure out how far in time from the root start
    # use that info to create a signal that is continuous from that start
    # going to presume there are 5820 seconds per orbit (97 minute period)
    time_delta = dates[0] - dt.datetime(2009, 1, 1)

    # mlt runs 0-24 each orbit.
    mlt = mm_test.generate_fake_data(time_delta.total_seconds(),
                                     uts,
                                     period=iperiod['lt'],
                                     data_range=drange['lt'])
    data['mlt'] = ((epoch_name), mlt)

    # do slt, 20 second offset from mlt
    slt = mm_test.generate_fake_data(time_delta.total_seconds() + 20,
                                     uts,
                                     period=iperiod['lt'],
                                     data_range=drange['lt'])
    data['slt'] = ((epoch_name), slt)

    # create a fake satellite longitude, resets every 6240 seconds
    # sat moves at 360/5820 deg/s, Earth rotates at 360/86400, takes extra time
    # to go around full longitude
    longitude = mm_test.generate_fake_data(time_delta.total_seconds(),
                                           uts,
                                           period=iperiod['lon'],
                                           data_range=drange['lon'])
    data['longitude'] = ((epoch_name), longitude)

    # create fake satellite latitude for testing polar orbits
    angle = mm_test.generate_fake_data(time_delta.total_seconds(),
                                       uts,
                                       period=iperiod['angle'],
                                       data_range=drange['angle'])
    latitude = 90.0 * np.cos(angle)
    data['latitude'] = ((epoch_name), latitude)

    # create constant altitude at 400 km for a satellite that has yet
    # to experience orbital decay
    alt0 = 400.0
    altitude = alt0 * np.ones(data['latitude'].shape)
    data['altitude'] = ((epoch_name), altitude)

    # create some fake data to support testing of averaging routines
    mlt_int = data['mlt'].astype(int).data
    long_int = (data['longitude'] / 15.).astype(int).data
    data['dummy1'] = ((epoch_name), mlt_int)
    data['dummy2'] = ((epoch_name), long_int)
    data['dummy3'] = ((epoch_name), mlt_int + long_int * 1000.)
    data['dummy4'] = ((epoch_name), uts)

    # Add dummy coords
    data.coords['x'] = (('x'), np.arange(17))
    data.coords['y'] = (('y'), np.arange(17))
    data.coords['z'] = (('z'), np.arange(15))

    # create altitude 'profile' at each location to simulate remote data
    num = len(data['uts'])
    data['profiles'] = ((epoch_name, 'profile_height'),
                        data['dummy3'].values[:, np.newaxis] * np.ones(
                            (num, 15)))
    data.coords['profile_height'] = ('profile_height', np.arange(15))

    # profiles that could have different altitude values
    data['variable_profiles'] = ((epoch_name,
                                  'z'), data['dummy3'].values[:, np.newaxis] *
                                 np.ones((num, 15)))
    data.coords['variable_profile_height'] = ((epoch_name, 'z'),
                                              np.arange(15)[np.newaxis, :] *
                                              np.ones((num, 15)))

    # Create fake image type data, projected to lat / lon at some location
    # from satellite
    data['images'] = ((epoch_name, 'x',
                       'y'), data['dummy3'].values[:, np.newaxis, np.newaxis] *
                      np.ones((num, 17, 17)))
    data.coords['image_lat'] = \
        ((epoch_name, 'x', 'y'),
         np.arange(17)[np.newaxis,
                       np.newaxis,
                       :] * np.ones((num, 17, 17)))
    data.coords['image_lon'] = ((epoch_name, 'x', 'y'),
                                np.arange(17)[np.newaxis, np.newaxis, :] *
                                np.ones((num, 17, 17)))

    # create very limited metadata
    meta = pysat.Meta()
    meta[epoch_name] = {'long_name': 'Datetime Index'}
    meta['uts'] = {'units': 's', 'long_name': 'Universal Time'}
    meta['mlt'] = {'units': 'hours', 'long_name': 'Magnetic Local Time'}
    meta['slt'] = {'units': 'hours', 'long_name': 'Solar Local Time'}
    meta['longitude'] = {'units': 'degrees', 'long_name': 'Longitude'}
    meta['latitude'] = {'units': 'degrees', 'long_name': 'Latitude'}
    meta['altitude'] = {'units': 'km', 'long_name': 'Altitude'}
    variable_profile_meta = pysat.Meta()
    variable_profile_meta['variable_profiles'] = {'long_name': 'series'}
    meta['variable_profiles'] = {
        'meta': variable_profile_meta,
        'long_name': 'series'
    }
    profile_meta = pysat.Meta()
    profile_meta['density'] = {'long_name': 'profiles'}
    profile_meta['dummy_str'] = {'long_name': 'profiles'}
    profile_meta['dummy_ustr'] = {'long_name': 'profiles'}
    meta['profiles'] = {'meta': profile_meta, 'long_name': 'profiles'}
    image_meta = pysat.Meta()
    image_meta['density'] = {'long_name': 'profiles'}
    image_meta['fraction'] = {'long_name': 'profiles'}
    meta['images'] = {'meta': image_meta, 'long_name': 'profiles'}
    for var in data.keys():
        if var.find('dummy') >= 0:
            meta[var] = {
                'units': 'none',
                'long_name': var,
                'notes': 'Dummy variable'
            }
    meta['x'] = {
        'long_name': 'x-value of image pixel',
        'notes': 'Dummy Variable'
    }
    meta['y'] = {
        'long_name': 'y-value of image pixel',
        'notes': 'Dummy Variable'
    }
    meta['z'] = {
        'long_name': 'z-value of profile height',
        'notes': 'Dummy Variable'
    }
    meta['image_lat'] = {
        'long_name': 'Latitude of image pixel',
        'notes': 'Dummy Variable'
    }
    meta['image_lon'] = {
        'long_name': 'Longitude of image pixel',
        'notes': 'Dummy Variable'
    }
    meta['profile_height'] = {'long_name': 'profile height'}
    meta['variable_profile_height'] = {'long_name': 'Variable Profile Height'}

    return data, meta
Ejemplo n.º 4
0
def load(fnames, tag=None, sat_id=None, sim_multi_file_right=False,
         sim_multi_file_left=False, malformed_index=False,
         **kwargs):
    """ Loads the test files

    Parameters
    ----------
    fnames : list
        List of filenames
    tag : str or NoneType
        Instrument tag (accepts '')
    sat_id : str or NoneType
        Instrument satellite ID (accepts '' or a number (i.e., '10'), which
        specifies the number of data points to include in the test instrument)
    sim_multi_file_right : boolean
        Adjusts date range to be 12 hours in the future or twelve hours beyond
        root_date (default=False)
    sim_multi_file_left : boolean
        Adjusts date range to be 12 hours in the past or twelve hours before
        root_date (default=False)
    malformed_index : boolean
        If True, time index will be non-unique and non-monotonic.
    kwargs : dict
        Additional unspecified keywords supplied to pysat.Instrument upon
        instantiation are passed here.

    Returns
    -------
    data : (xr.Dataset)
        Testing data
    meta : (pysat.Meta)
        Metadata

    """

    # create an artifical satellite data set
    iperiod = mm_test.define_period()
    drange = mm_test.define_range()
    uts, index, date = mm_test.generate_times(fnames, sat_id=sat_id, freq='1S')

    if sim_multi_file_right:
        root_date = pysat.datetime(2009, 1, 1, 12)
    elif sim_multi_file_left:
        root_date = pysat.datetime(2008, 12, 31, 12)
    else:
        root_date = pysat.datetime(2009, 1, 1)

    if malformed_index:
        index = index.tolist()
        # nonmonotonic
        index[0:3], index[3:6] = index[3:6], index[0:3]
        # non unique
        index[6:9] = [index[6]]*3

    data = xarray.Dataset({'uts': (('time'), index)}, coords={'time': index})
    # need to create simple orbits here. Have start of first orbit
    # at 2009,1, 0 UT. 14.84 orbits per day
    time_delta = date - root_date
    mlt = mm_test.generate_fake_data(time_delta.total_seconds(), uts,
                                     period=iperiod['lt'],
                                     data_range=drange['lt'])
    data['mlt'] = (('time'), mlt)

    # do slt, 20 second offset from mlt
    slt = mm_test.generate_fake_data(time_delta.total_seconds()+20, uts,
                                     period=iperiod['lt'],
                                     data_range=drange['lt'])
    data['slt'] = (('time'), slt)

    # create a fake longitude, resets every 6240 seconds
    # sat moves at 360/5820 deg/s, Earth rotates at 360/86400, takes extra time
    # to go around full longitude
    longitude = mm_test.generate_fake_data(time_delta.total_seconds(), uts,
                                           period=iperiod['lon'],
                                           data_range=drange['lon'])
    data['longitude'] = (('time'), longitude)

    # create latitude area for testing polar orbits
    angle = mm_test.generate_fake_data(time_delta.total_seconds(), uts,
                                       period=iperiod['angle'],
                                       data_range=drange['angle'])
    latitude = 90.0 * np.cos(angle)
    data['latitude'] = (('time'), latitude)

    # fake orbit number
    fake_delta = date - pysat.datetime(2008, 1, 1)
    orbit_num = mm_test.generate_fake_data(fake_delta.total_seconds(),
                                           uts, period=iperiod['lt'],
                                           cyclic=False)

    data['orbit_num'] = (('time'), orbit_num)

    # create some fake data to support testing of averaging routines
    mlt_int = data['mlt'].astype(int)
    long_int = (data['longitude'] / 15.).astype(int)
    data['dummy1'] = (('time'), mlt_int)
    data['dummy2'] = (('time'), long_int)
    data['dummy3'] = (('time'), mlt_int + long_int * 1000.)
    data['dummy4'] = (('time'), uts)
    data['string_dummy'] = (('time'), ['test'] * len(data.indexes['time']))
    data['unicode_dummy'] = (('time'), [u'test'] * len(data.indexes['time']))
    data['int8_dummy'] = (('time'), np.array([1] * len(data.indexes['time']),
                          dtype=np.int8))
    data['int16_dummy'] = (('time'), np.array([1] * len(data.indexes['time']),
                           dtype=np.int16))
    data['int32_dummy'] = (('time'), np.array([1] * len(data.indexes['time']),
                           dtype=np.int32))
    data['int64_dummy'] = (('time'), np.array([1] * len(data.indexes['time']),
                           dtype=np.int64))

    return data, meta.copy()
Ejemplo n.º 5
0
def load(fnames, tag=None, sat_id=None, malformed_index=False):
    """ Loads the test files

    Parameters
    ----------
    fnames : list
        List of filenames
    tag : str or NoneType
        Instrument tag (accepts '')
    sat_id : str or NoneType
        Instrument satellite ID (accepts '' or a number (i.e., '10'), which
        specifies the number of data points to include in the test instrument)
    malformed_index : bool
        If True, the time index will be non-unique and non-monotonic.
        (default=False)

    Returns
    -------
    data : pds.DataFrame
        Testing data
    meta : pysat.Meta
        Metadataxs

    """

    # create an artifical satellite data set
    iperiod = mm_test.define_period()
    drange = mm_test.define_range()
    # Using 100s frequency for compatibility with seasonal analysis unit tests
    uts, index, date = mm_test.generate_times(fnames, sat_id, freq='100S')
    # seed DataFrame with UT array
    data = pysat.DataFrame(uts, columns=['uts'])

    # need to create simple orbits here. Have start of first orbit
    # at 2009,1, 0 UT. 14.84 orbits per day
    # figure out how far in time from the root start
    # use that info to create a signal that is continuous from that start
    # going to presume there are 5820 seconds per orbit (97 minute period)
    time_delta = date - pysat.datetime(2009, 1, 1)
    # mlt runs 0-24 each orbit.
    data['mlt'] = mm_test.generate_fake_data(time_delta.total_seconds(),
                                             uts,
                                             period=iperiod['lt'],
                                             data_range=drange['lt'])
    # do slt, 20 second offset from mlt
    data['slt'] = mm_test.generate_fake_data(time_delta.total_seconds() + 20,
                                             uts,
                                             period=iperiod['lt'],
                                             data_range=drange['lt'])
    # create a fake longitude, resets every 6240 seconds
    # sat moves at 360/5820 deg/s, Earth rotates at 360/86400, takes extra time
    # to go around full longitude
    data['longitude'] = mm_test.generate_fake_data(time_delta.total_seconds(),
                                                   uts,
                                                   period=iperiod['lon'],
                                                   data_range=drange['lon'])
    # create latitude signal for testing polar orbits
    angle = mm_test.generate_fake_data(time_delta.total_seconds(),
                                       uts,
                                       period=iperiod['angle'],
                                       data_range=drange['angle'])
    data['latitude'] = 90.0 * np.cos(angle)

    if malformed_index:
        index = index.tolist()
        # nonmonotonic
        index[0:3], index[3:6] = index[3:6], index[0:3]
        # non unique
        index[6:9] = [index[6]] * 3

    data.index = index
    data.index.name = 'epoch'
    # higher rate time signal (for scalar >= 2)
    # this time signal used for 2D profiles associated with each time in main
    # DataFrame
    high_rate_template = pds.date_range(
        date, date + pds.DateOffset(hours=0, minutes=1, seconds=39), freq='2S')

    # create a few simulated profiles
    # DataFrame at each time with mixed variables
    profiles = []
    # DataFrame at each time with numeric variables only
    alt_profiles = []
    # Serie at each time, numeric data only
    series_profiles = []
    # frame indexed by date times
    frame = pds.DataFrame(
        {
            'density': data.loc[data.index[0:50], 'mlt'].values.copy(),
            'dummy_str': ['test'] * 50,
            'dummy_ustr': [u'test'] * 50
        },
        index=data.index[0:50],
        columns=['density', 'dummy_str', 'dummy_ustr'])
    # frame indexed by float
    dd = np.arange(50) * 1.2
    ff = np.arange(50) / 50.
    ii = np.arange(50) * 0.5
    frame_alt = pds.DataFrame({
        'density': dd,
        'fraction': ff
    },
                              index=ii,
                              columns=['density', 'fraction'])
    # series version of storage
    series_alt = pds.Series(dd, index=ii, name='series_profiles')

    for time in data.index:
        frame.index = high_rate_template + (time - data.index[0])
        profiles.append(frame)
        alt_profiles.append(frame_alt)
        series_profiles.append(series_alt)
    # store multiple data types into main frame
    data['profiles'] = pds.Series(profiles, index=data.index)
    data['alt_profiles'] = pds.Series(alt_profiles, index=data.index)
    data['series_profiles'] = pds.Series(series_profiles, index=data.index)
    return data, meta.copy()
Ejemplo n.º 6
0
def load(fnames, tag=None, inst_id=None, sim_multi_file_right=False,
         sim_multi_file_left=False, malformed_index=False,
         num_samples=None, test_load_kwarg=None):
    """ Loads the test files

    Parameters
    ----------
    fnames : list
        List of filenames
    tag : str or NoneType
        Instrument tag (accepts '')
    inst_id : str or NoneType
        Instrument satellite ID (accepts '')
    sim_multi_file_right : boolean
        Adjusts date range to be 12 hours in the future or twelve hours beyond
        root_date (default=False)
    sim_multi_file_left : boolean
        Adjusts date range to be 12 hours in the past or twelve hours before
        root_date (default=False)
    malformed_index : boolean
        If True, time index will be non-unique and non-monotonic.
    num_samples : int
        Number of samples
    test_load_kwarg : any or NoneType
        Testing keyword (default=None)

    Returns
    -------
    data : xr.Dataset
        Testing data
    meta : pysat.Meta
        Metadata

    """

    # Support keyword testing
    logger.info(''.join(('test_load_kwarg = ', str(test_load_kwarg))))

    # create an artifical satellite data set
    iperiod = mm_test.define_period()
    drange = mm_test.define_range()

    if num_samples is None:
        # Default to 1 day at a frequency of 1S
        num_samples = 86400
    uts, index, dates = mm_test.generate_times(fnames, num_samples,
                                               freq='1S')

    if sim_multi_file_right:
        root_date = dt.datetime(2009, 1, 1, 12)
    elif sim_multi_file_left:
        root_date = dt.datetime(2008, 12, 31, 12)
    else:
        root_date = dt.datetime(2009, 1, 1)

    if malformed_index:
        index = index.tolist()

        # Create a nonmonotonic index
        index[0:3], index[3:6] = index[3:6], index[0:3]

        # Create a non-unique index
        index[6:9] = [index[6]] * 3

    data = xr.Dataset({'uts': ((epoch_name), index)},
                      coords={epoch_name: index})
    # need to create simple orbits here. Have start of first orbit
    # at 2009,1, 0 UT. 14.84 orbits per day
    time_delta = dates[0] - root_date
    mlt = mm_test.generate_fake_data(time_delta.total_seconds(), uts,
                                     period=iperiod['lt'],
                                     data_range=drange['lt'])
    data['mlt'] = ((epoch_name), mlt)

    # do slt, 20 second offset from mlt
    slt = mm_test.generate_fake_data(time_delta.total_seconds() + 20, uts,
                                     period=iperiod['lt'],
                                     data_range=drange['lt'])
    data['slt'] = ((epoch_name), slt)

    # create a fake longitude, resets every 6240 seconds
    # sat moves at 360/5820 deg/s, Earth rotates at 360/86400, takes extra time
    # to go around full longitude
    longitude = mm_test.generate_fake_data(time_delta.total_seconds(), uts,
                                           period=iperiod['lon'],
                                           data_range=drange['lon'])
    data['longitude'] = ((epoch_name), longitude)

    # create latitude area for testing polar orbits
    angle = mm_test.generate_fake_data(time_delta.total_seconds(), uts,
                                       period=iperiod['angle'],
                                       data_range=drange['angle'])
    latitude = 90.0 * np.cos(angle)
    data['latitude'] = ((epoch_name), latitude)

    # create constant altitude at 400 km
    alt0 = 400.0
    altitude = alt0 * np.ones(data['latitude'].shape)
    data['altitude'] = ((epoch_name), altitude)

    # fake orbit number
    fake_delta = dates[0] - dt.datetime(2008, 1, 1)
    orbit_num = mm_test.generate_fake_data(fake_delta.total_seconds(),
                                           uts, period=iperiod['lt'],
                                           cyclic=False)

    data['orbit_num'] = ((epoch_name), orbit_num)

    # create some fake data to support testing of averaging routines
    mlt_int = data['mlt'].astype(int).data
    long_int = (data['longitude'] / 15.).astype(int).data
    data['dummy1'] = ((epoch_name), mlt_int)
    data['dummy2'] = ((epoch_name), long_int)
    data['dummy3'] = ((epoch_name), mlt_int + long_int * 1000.)
    data['dummy4'] = ((epoch_name), uts)
    data['string_dummy'] = ((epoch_name),
                            ['test'] * len(data.indexes[epoch_name]))
    data['unicode_dummy'] = ((epoch_name),
                             [u'test'] * len(data.indexes[epoch_name]))
    data['int8_dummy'] = ((epoch_name),
                          np.array([1] * len(data.indexes[epoch_name]),
                          dtype=np.int8))
    data['int16_dummy'] = ((epoch_name),
                           np.array([1] * len(data.indexes[epoch_name]),
                           dtype=np.int16))
    data['int32_dummy'] = ((epoch_name),
                           np.array([1] * len(data.indexes[epoch_name]),
                           dtype=np.int32))
    data['int64_dummy'] = ((epoch_name),
                           np.array([1] * len(data.indexes[epoch_name]),
                           dtype=np.int64))

    meta = pysat.Meta()
    meta['uts'] = {'units': 's', 'long_name': 'Universal Time',
                   'custom': False}
    meta[epoch_name] = {'units': 'Milliseconds since 1970-1-1',
                        'Bin_Location': 0.5,
                        'notes':
                        'UTC time at middle of geophysical measurement.',
                        'desc': 'UTC seconds', }
    meta['mlt'] = {'units': 'hours',
                   'long_name': 'Magnetic Local Time',
                   'desc': 'Magnetic Local Time',
                   'value_min': 0.,
                   'value_max': 24.,
                   'notes': ''.join(['Magnetic Local Time is the solar local ',
                                     'time of the field line at the location ',
                                     'where the field crosses the magnetic ',
                                     'equator. In this case we just simulate ',
                                     '0-24 with a consistent orbital period ',
                                     'and an offset with SLT.'])}
    meta['slt'] = {'units': 'hours', 'long_name': 'Solar Local Time',
                   'desc': 'Solar Local Time', 'value_min': 0.,
                   'value_max': 24.,
                   'notes': ''.join(['Solar Local Time is the local time ',
                                     '(zenith angle of thee sun) of the given',
                                     ' locaiton. Overhead noon, +/- 90 is 6,',
                                     ' 18 SLT .'])}
    meta['orbit_num'] = {'long_name': 'Orbit Number', 'desc': 'Orbit Number',
                         'value_min': 0., 'value_max': 25000.,
                         'notes': ''.join(['Number of orbits since the start ',
                                           'of the mission. For this ',
                                           'simulation we use the ',
                                           'number of 5820 second periods ',
                                           'since the start, 2008-01-01.'])}

    meta['longitude'] = {'units': 'degrees', 'long_name': 'Longitude'}
    meta['latitude'] = {'units': 'degrees', 'long_name': 'Latitude'}
    meta['altitude'] = {'units': 'km', 'long_name': 'Altitude'}
    for var in data.keys():
        if var.find('dummy') >= 0:
            meta[var] = {'units': 'none', 'notes': 'Dummy variable'}

    return data, meta
Ejemplo n.º 7
0
def load(fnames, tag=None, sat_id=None, malformed_index=False):
    """ Loads the test files

    Parameters
    ----------
    fnames : list
        List of filenames
    tag : str or NoneType
        Instrument tag (accepts '')
    sat_id : str or NoneType
        Instrument satellite ID (accepts '' or a number (i.e., '10'), which
        specifies the number of data points to include in the test instrument)
    malformed_index : bool False
        If True, the time index will be non-unique and non-monotonic.

    Returns
    -------
    data : xr.Dataset
        Testing data
    meta : pysat.Meta
        Metadataxs

    """

    # create an artifical satellite data set
    iperiod = mm_test.define_period()
    drange = mm_test.define_range()
    # Using 100s frequency for compatibility with seasonal analysis unit tests
    uts, index, date = mm_test.generate_times(fnames, sat_id, freq='100S')

    if malformed_index:
        index = index.tolist()
        # nonmonotonic
        index[0:3], index[3:6] = index[3:6], index[0:3]
        # non unique
        index[6:9] = [index[6]]*3
    data = xr.Dataset({'uts': (('time'), index)}, coords={'time': index})

    # need to create simple orbits here. Have start of first orbit
    # at 2009,1, 0 UT. 14.84 orbits per day
    # figure out how far in time from the root start
    # use that info to create a signal that is continuous from that start
    # going to presume there are 5820 seconds per orbit (97 minute period)
    time_delta = date - pysat.datetime(2009, 1, 1)

    # mlt runs 0-24 each orbit.
    mlt = mm_test.generate_fake_data(time_delta.total_seconds(), uts,
                                     period=iperiod['lt'],
                                     data_range=drange['lt'])
    data['mlt'] = (('time'), mlt)

    # do slt, 20 second offset from mlt
    slt = mm_test.generate_fake_data(time_delta.total_seconds()+20, uts,
                                     period=iperiod['lt'],
                                     data_range=drange['lt'])
    data['slt'] = (('time'), slt)

    # create a fake longitude, resets every 6240 seconds
    # sat moves at 360/5820 deg/s, Earth rotates at 360/86400, takes extra time
    # to go around full longitude
    longitude = mm_test.generate_fake_data(time_delta.total_seconds(), uts,
                                           period=iperiod['lon'],
                                           data_range=drange['lon'])
    data['longitude'] = (('time'), longitude)

    # create latitude signal for testing polar orbits
    angle = mm_test.generate_fake_data(time_delta.total_seconds(), uts,
                                       period=iperiod['angle'],
                                       data_range=drange['angle'])
    latitude = 90.0 * np.cos(angle)
    data['latitude'] = (('time'), latitude)

    # create some fake data to support testing of averaging routines
    mlt_int = data['mlt'].astype(int)
    long_int = (data['longitude'] / 15.).astype(int)
    data['dummy1'] = (('time'), mlt_int)
    data['dummy2'] = (('time'), long_int)
    data['dummy3'] = (('time'), mlt_int + long_int * 1000.)
    data['dummy4'] = (('time'), uts)

    # create altitude 'profile' at each location
    num = len(data['uts'])
    data['profiles'] = \
        (('time', 'altitude'),
         data['dummy3'].values[:, np.newaxis] * np.ones((num, 15)))
    data.coords['altitude'] = ('altitude', np.arange(15))

    # profiles that could have different altitude values
    data['variable_profiles'] = \
        (('time', 'z'),
         data['dummy3'].values[:, np.newaxis] * np.ones((num, 15)))
    data.coords['altitude2'] = \
        (('time', 'z'),
         np.arange(15)[np.newaxis, :]*np.ones((num, 15)))

    # basic image simulation
    data['images'] = \
        (('time', 'x', 'y'),
         data['dummy3'].values[:,
                               np.newaxis,
                               np.newaxis] * np.ones((num, 17, 17)))
    data.coords['latitude'] = \
        (('time', 'x', 'y'),
         np.arange(17)[np.newaxis,
                       np.newaxis,
                       :]*np.ones((num, 17, 17)))
    data.coords['longitude'] = \
        (('time', 'x', 'y'),
         np.arange(17)[np.newaxis,
                       np.newaxis,
                       :] * np.ones((num, 17, 17)))

    return data, meta.copy()
Ejemplo n.º 8
0
def load(fnames,
         tag=None,
         inst_id=None,
         sim_multi_file_right=False,
         sim_multi_file_left=False,
         root_date=None,
         malformed_index=False,
         num_samples=None,
         test_load_kwarg=None):
    """ Loads the test files

    Parameters
    ----------
    fnames : list
        List of filenames
    tag : str or NoneType
        Instrument tag (accepts '' or a string to change the behaviour of
        certain instrument aspects for testing)
    inst_id : str or NoneType
        Instrument satellite ID (accepts '')
    sim_multi_file_right : boolean
        Adjusts date range to be 12 hours in the future or twelve hours beyond
        root_date (default=False)
    sim_multi_file_left : boolean
        Adjusts date range to be 12 hours in the past or twelve hours before
        root_date (default=False)
    root_date : NoneType
        Optional central date, uses _test_dates if not specified.
        (default=None)
    malformed_index : boolean
        If True, time index will be non-unique and non-monotonic (default=False)
    num_samples : int
        Number of samples per day
    test_load_kwarg : any or NoneType
        Testing keyword (default=None)

    Returns
    -------
    data : pds.DataFrame
        Testing data
    meta : pysat.Meta
        Metadata

    """

    # Support keyword testing
    logger.info(''.join(('test_load_kwarg = ', str(test_load_kwarg))))

    # create an artificial satellite data set
    iperiod = mm_test.define_period()
    drange = mm_test.define_range()

    if num_samples is None:
        # Default to 1 day at a frequency of 1S
        num_samples = 86400
    uts, index, dates = mm_test.generate_times(fnames, num_samples, freq='1S')

    # Specify the date tag locally and determine the desired date range
    pds_offset = dt.timedelta(hours=12)
    if sim_multi_file_right:
        root_date = root_date or _test_dates[''][''] + pds_offset
    elif sim_multi_file_left:
        root_date = root_date or _test_dates[''][''] - pds_offset
    else:
        root_date = root_date or _test_dates['']['']

    # store UTS, mod 86400
    data = pds.DataFrame(np.mod(uts, 86400.), columns=['uts'])

    # need to create simple orbits here. Have start of first orbit default
    # to 1 Jan 2009, 00:00 UT. 14.84 orbits per day
    time_delta = dates[0] - root_date
    data['mlt'] = mm_test.generate_fake_data(time_delta.total_seconds(),
                                             uts,
                                             period=iperiod['lt'],
                                             data_range=drange['lt'])

    # do slt, 20 second offset from mlt
    data['slt'] = mm_test.generate_fake_data(time_delta.total_seconds() + 20,
                                             uts,
                                             period=iperiod['lt'],
                                             data_range=drange['lt'])

    # create a fake longitude, resets every 6240 seconds
    # sat moves at 360/5820 deg/s, Earth rotates at 360/86400, takes extra time
    # to go around full longitude
    data['longitude'] = mm_test.generate_fake_data(time_delta.total_seconds(),
                                                   uts,
                                                   period=iperiod['lon'],
                                                   data_range=drange['lon'])

    # create latitude area for testing polar orbits
    angle = mm_test.generate_fake_data(time_delta.total_seconds(),
                                       uts,
                                       period=iperiod['angle'],
                                       data_range=drange['angle'])
    data['latitude'] = 90.0 * np.cos(angle)

    # create constant altitude at 400 km
    alt0 = 400.0
    data['altitude'] = alt0 * np.ones(data['latitude'].shape)

    # fake orbit number
    fake_delta = dates[0] - (_test_dates[''][''] - pds.DateOffset(years=1))
    data['orbit_num'] = mm_test.generate_fake_data(fake_delta.total_seconds(),
                                                   uts,
                                                   period=iperiod['lt'],
                                                   cyclic=False)

    # create some fake data to support testing of averaging routines
    mlt_int = data['mlt'].astype(int)
    long_int = (data['longitude'] / 15.0).astype(int)
    data['dummy1'] = mlt_int
    data['dummy2'] = long_int
    data['dummy3'] = mlt_int + long_int * 1000.0
    data['dummy4'] = uts
    data['string_dummy'] = ['test'] * len(data)
    data['unicode_dummy'] = [u'test'] * len(data)
    data['int8_dummy'] = np.ones(len(data), dtype=np.int8)
    data['int16_dummy'] = np.ones(len(data), dtype=np.int16)
    data['int32_dummy'] = np.ones(len(data), dtype=np.int32)
    data['int64_dummy'] = np.ones(len(data), dtype=np.int64)

    # Activate for testing malformed_index, and for instrument_test_class
    if malformed_index or tag == 'non_strict':
        index = index.tolist()
        # nonmonotonic
        index[0:3], index[3:6] = index[3:6], index[0:3]
        # non unique
        index[6:9] = [index[6]] * 3

    data.index = index
    data.index.name = 'Epoch'

    # Set the meta data
    meta = pysat.Meta()
    meta['uts'] = {
        'units': 's',
        'long_name': 'Universal Time',
        'custom': False
    }
    meta['Epoch'] = {
        'units': 'Milliseconds since 1970-1-1',
        'Bin_Location': 0.5,
        'notes': 'UTC time at middle of geophysical measurement.',
        'desc': 'UTC seconds'
    }
    meta['mlt'] = {
        'units':
        'hours',
        'long_name':
        'Magnetic Local Time',
        'desc':
        'Magnetic Local Time',
        'value_min':
        0.0,
        'value_max':
        24.0,
        'notes':
        ''.join([
            'Magnetic Local Time is the solar local ',
            'time of thefield line at the location ',
            'where the field crosses the magnetic ',
            'equator. In this case we just simulate ',
            '0-24 with a consistent orbital period ', 'and an offset with SLT.'
        ]),
        'scale':
        'linear'
    }
    meta['slt'] = {
        'units':
        'hours',
        'long_name':
        'Solar Local Time',
        'desc':
        'Solar Local Time',
        'value_min':
        0.0,
        'value_max':
        24.0,
        'notes':
        ''.join([
            'Solar Local Time is the local time ',
            '(zenith angle of sun) of the given ',
            'location. Overhead noon, +/- 90 is 6, ', '18 SLT .'
        ])
    }
    meta['orbit_num'] = {
        'units':
        '',
        'long_name':
        'Orbit Number',
        'desc':
        'Orbit Number',
        'value_min':
        0.0,
        'value_max':
        25000.0,
        'notes':
        ''.join([
            'Number of orbits since the start ', 'of the mission. For this ',
            'simulation we use the number of ',
            '5820 second periods since the ', 'start, 2008-01-01.'
        ])
    }
    meta['longitude'] = {'units': 'degrees', 'long_name': 'Longitude'}
    meta['latitude'] = {'units': 'degrees', 'long_name': 'Latitude'}
    meta['altitude'] = {'units': 'km', 'long_name': 'Altitude'}
    if tag != 'default_meta':
        for var in data.keys():
            if var.find('dummy') >= 0:
                meta[var] = {
                    'units': 'none',
                    'notes': 'Dummy variable for testing'
                }

    return data, meta