def get_model_signature_lists(runprops,
epoch=datetime(2005, 1, 1, tzinfo=UTC),
datadir='.'):
onset_lists = {}
auroral_inds = dm.fromHDF5(
os.path.join(
datadir, runprops['name'].replace('/', '').replace(' ', '_') +
'_auroral_inds.h5'))
onsets_borovsky = borovsky_id_algorithm(auroral_inds['AL'])
onsets_borovsky = [
datetime(2005, 1, 1, tzinfo=UTC) + timedelta(minutes=m)
for m in onsets_borovsky
]
onset_lists['AL'] = get_tnums(onsets_borovsky, epoch)
plasmoid_file = os.path.join(
datadir, 'plasmoids_' +
runprops['name'].replace('/', '').replace(' ', '_') + '.h5')
if os.path.isfile(plasmoid_file):
plasmoid_data = dm.fromHDF5(plasmoid_file)
plasmoid_times = np.array([
datetime.strptime(s, '%Y-%m-%dT%H:%M:%S').replace(tzinfo=UTC)
for s in plasmoid_data['time']
])
plasmoid_times = plasmoid_times[plasmoid_data['x'] > -35]
else:
plasmoid_times = []
onset_lists['plasmoids'] = get_tnums(plasmoid_times, epoch)
#midn_distances=(3,5,7,10,)#15,20,30,40,50)
midn_distances = [7]
dipolarizations = list(
itertools.chain.from_iterable([
get_dipolarizations(runprops['name'], satname, datadir)
for satname in ['goes10', 'goes12'] +
['midn_{0:02d}'.format(distance) for distance in midn_distances]
]))
dipolarizations.sort()
onset_lists['dipolarizations'] = get_tnums(dipolarizations, epoch)
namestr = runprops['name'].replace('/', '').replace(' ', '_')
mpb_onset_file = glob(
os.path.join(
datadir, 'John Haiducek - ' + namestr +
'_mag_grid_lat=4?.matonset_tmax.txt'))
if len(mpb_onset_file) == 1:
onset, tmax = parse_onset_tmax(mpb_onset_file[0])
else:
onset = []
onset_lists['MPB'] = get_tnums(onset, epoch)
return onset_lists
def test_HDF5roundtripGZIP(self):
"""Data can go to hdf and back with compression"""
dm.toHDF5(self.testfile, self.SDobj, compression='gzip')
newobj = dm.fromHDF5(self.testfile)
self.assertEqual(self.SDobj.attrs['global'], newobj.attrs['global'])
np.testing.assert_almost_equal(self.SDobj['var'], newobj['var'])
self.assertEqual(self.SDobj['var'].attrs['a'], newobj['var'].attrs['a'])
dm.toHDF5(self.testfile, self.SDobj, mode='a', compression='gzip')
newobj = dm.fromHDF5(self.testfile)
self.assertEqual(self.SDobj.attrs['global'], newobj.attrs['global'])
np.testing.assert_almost_equal(self.SDobj['var'], newobj['var'])
self.assertEqual(self.SDobj['var'].attrs['a'], newobj['var'].attrs['a'])
def test_HDF5roundtrip_method(self):
"""Data can go to hdf and back"""
self.SDobj.toHDF5(self.testfile)
newobj = dm.fromHDF5(self.testfile)
self.assertEqual(self.SDobj.attrs['global'], newobj.attrs['global'])
np.testing.assert_almost_equal(self.SDobj['var'], newobj['var'])
self.assertEqual(self.SDobj['var'].attrs['a'], newobj['var'].attrs['a'])
dm.toHDF5(self.testfile, self.SDobj, mode='a')
newobj = dm.fromHDF5(self.testfile)
self.assertEqual(self.SDobj.attrs['global'], newobj.attrs['global'])
np.testing.assert_almost_equal(self.SDobj['var'], newobj['var'])
self.assertEqual(self.SDobj['var'].attrs['a'], newobj['var'].attrs['a'])
def test_toHDF5ListString(self):
"""Convert to HDF5, including a list of string in attributes"""
a = dm.SpaceData()
a.attrs['foo'] = ['hi']
dm.toHDF5(self.testfile, a, mode='a')
newobj = dm.fromHDF5(self.testfile)
self.assertEqual(a.attrs['foo'], newobj.attrs['foo'])
def test_copy_toHDF(self):
"""Removed code in #404 for an old python bug, issue4380, make sure it works"""
a = dm.SpaceData({'dat': dm.dmarray([1, 2, 3])})
a2 = copy.deepcopy(a)
a2.toHDF5(self.testfile, mode='a')
newobj = dm.fromHDF5(self.testfile)
np.testing.assert_array_equal([1, 2, 3], newobj['dat'])
def get_dipolarizations(run_name, satname, datadir='.'):
namestr = run_name.replace('/', '').replace(' ', '_')
satdata = dm.fromHDF5(
os.path.join(datadir, namestr + '_' + satname + '.h5'))
times = np.array(num2date(satdata['numtime']))
mlt = (np.arctan2(-satdata['Y'], -satdata['X']) * 12 / np.pi) % 24
newtimes = np.array([
datetime(2005, 1, 1, tzinfo=UTC) + timedelta(seconds=s * 60)
for s in range(1440 * 31)
])
bx = interp_timeseries(satdata['bx'], times, newtimes)
by = interp_timeseries(satdata['by'], times, newtimes)
bz = interp_timeseries(satdata['bz'], times, newtimes)
x = interp_timeseries(satdata['X'], times, newtimes)
y = interp_timeseries(satdata['Y'], times, newtimes)
mlt = interp_timeseries(mlt, times, newtimes)
times = newtimes
br = (bx * x + by * y) / np.sqrt(x**2 + y**2)
theta = np.arctan2(bz, np.sqrt(bx**2 + by**2)) * 180 / np.pi
dip_inds = find_dipolarizations_br_bz_theta(newtimes, br, bz, theta)
night_inds = np.where((mlt < 6) | (mlt > 18))[0]
dip_inds = np.intersect1d(dip_inds, night_inds)
dip_times = times[dip_inds]
satdata['time'] = newtimes
satdata['bx'] = bx
satdata['by'] = by
satdata['bz'] = bz
satdata['br'] = br
satdata['theta'] = theta
return dip_times
def data(orbit_no, show_avail, show_tree):
''' Gets the HEPD and MEPD data of the corresponding orbit number in the
data path in SpacePy datamodel format. Also capable of showing
available files and the selected file's file structure tree if
show_avail or show_tree is set to True, respectively. '''
files = get_files(DATA_PATH, show_avail)
HEPD_file_path, MEPD_file_path = get_file_paths(orbit_no, DATA_PATH, files)
print('Selected orbit number: ', str(orbit_no), '\n')
# Create SpacePy datamodel.
HEPD_data = dm.fromHDF5(HEPD_file_path)
MEPD_data = dm.fromHDF5(MEPD_file_path)
if show_tree == True:
file_tree(HEPD_data, MEPD_data)
HEPD = isssData(orbit_no, HEPD_data, GROUP1, DATASET1, DATASET2)
MEPD = isssData(orbit_no, MEPD_data, GROUP2, DATASET1, DATASET2)
return HEPD, MEPD
def plot_sea_onset_comparison(run_name,
var,
ax,
signature_types=signature_types):
if run_name == 'obs':
seadata['MPB'] = ['MPB ($nT^4$)', obs_mpb_v, obs_mpb_t]
seadata['al'] = ['AL (nT)', sml, supermag_times]
ylabel, data, times = seadata[var]
artists, labels = plot_onset_sea(obs_signatures,
obs_threshold,
data,
times,
ylabel,
ax,
signature_types=signature_types)
else:
names = [runprops['name'] for runprops in run_properties]
runprops = run_properties[names.index(run_name)]
run_signatures = get_model_signature_lists(runprops, datadir=datadir)
from spacepy import datamodel as dm
auroral_inds = dm.fromHDF5(
os.path.join(
datadir, runprops['name'].replace('/', '').replace(' ', '_') +
'_auroral_inds.h5'))
al_time = [
datetime(2005, 1, 1, tzinfo=UTC) + timedelta(seconds=60 * m)
for m in range(0, 1440 * 31)
]
#try:
# mpbdata=loadmat(os.path.join(datadir,'John Haiducek - '+runprops['name'].replace('/','').replace(' ','_')+'_mag_grid_lat=33_mpb.mat'))
#except:
# raise
#mpb_t=[datetime(2005,1,1,tzinfo=UTC)+timedelta(seconds=m*60) for m in range(0,31*24*60)]
#mpb_v=mpbdata['mpb']
mpb_t, mpb_v = parse_index(os.path.join(datadir, 'mpb_index.txt'))
seadata['MPB'] = ['MPB ($nT^4$)', mpb_v, mpb_t]
seadata['al'] = ['AL (nT)', auroral_inds['AL'], al_time]
ylabel, data, times = seadata[var]
artists, labels = plot_onset_sea(run_signatures,
model_threshold,
data,
times,
ylabel,
ax,
signature_types=signature_types)
return artists, labels
def test_HDF5roundtrip2GZIP(self):
"""Data can go to hdf without altering datetimes in the datamodel with compression"""
a = dm.SpaceData()
a['foo'] = dm.SpaceData()
dm.toHDF5(self.testfile, a, compression='gzip')
newobj = dm.fromHDF5(self.testfile)
self.assertEqual(a['foo'], newobj['foo'])
a['bar'] = dm.dmarray([datetime.datetime(2000, 1, 1)])
dm.toHDF5(self.testfile, a, compression='gzip')
self.assertEqual(a['bar'], dm.dmarray([datetime.datetime(2000, 1, 1)]))
def fromHDF5(cls, filepath, **kwargs):
'''
Create a DataModel from a HDF5 data file.
Attributes are not available without SpacePy.
'''
if USE_SPACEPY:
return cls.fromSpaceData(spdm.fromHDF5(filepath, **kwargs))
else:
import h5py
data_model = cls()
path = '/'
if 'path' in kwargs:
path = kwargs['path']
h5file = h5py.File(filepath, mode='r')
for key in h5file[path]:
data_model[key] = numpy.array(h5file[path][key])
h5file.close()
return data_model
@cache_result()
def load_omni(dtstart, dtend, proxy=None):
return get_cdf('sp_phys',
'OMNI_HRO_1MIN',
dtstart,
dtend, [
'BX_GSE', 'BY_GSM', 'BZ_GSM', 'Vx', 'Vy', 'Vz', 'T',
'proton_density'
],
proxy=proxy)
# Read advect1d output
advect1d_data = dm.fromHDF5('advected.h5')
advect1d_data['time'] = advect1d_data['time'].astype('datetime64')
# Retrieve start/end times of data and rount to nearest second
starttime = advect1d_data['time'][0].astype('datetime64[s]').astype(datetime)
endtime = advect1d_data['time'][-1].astype('datetime64[s]').astype(datetime)
args = parse_args(starttime=starttime, endtime=endtime)
starttime = args.start_time
endtime = args.end_time
source = args.source
proxy = args.proxy
# Fetch OMNI data
omnidata = load_omni(starttime, endtime, proxy=proxy)
def test_toHDF5_method(self):
"""Convert to HDF5, using the method, catching #404"""
a = dm.SpaceData({'dat': dm.dmarray([1, 2, 3])})
a.toHDF5(self.testfile, mode='a')
newobj = dm.fromHDF5(self.testfile)
np.testing.assert_array_equal([1, 2, 3], newobj['dat'])
logfiles = []
for filename in filenames:
logfiles.append(SwmfLogFile(filename))
return LogSequence(logfiles)
if __name__ == '__main__':
cachefile = 'dipolarization_test.h5'
import os
from spacepy import datamodel as dm
from matplotlib.dates import date2num, num2date
if os.path.isfile(cachefile):
satdata = dm.fromHDF5(cachefile)
else:
satfiles = build_log_sequence(
'/data2/jhaiduce/substorms_Jan2005_young-comp/day*/GM/IO2/sat_goes10_20050101_n*.sat'
)
satdata = dm.SpaceData()
satdata['bz'] = dm.dmarray(satfiles['bz'])
satdata['time'] = dm.dmarray(date2num(satfiles['time']))
satdata.toHDF5(cachefile)
from matplotlib import pyplot as plt
dt = (satdata['time'][1:] - satdata['time'][:-1]) * 3600 * 24
satdata['time'] = np.array(num2date(satdata['time']))
