def get_model_signature_lists(runprops, epoch=datetime(2005, 1, 1, tzinfo=UTC), datadir='.'): onset_lists = {} auroral_inds = dm.fromHDF5( os.path.join( datadir, runprops['name'].replace('/', '').replace(' ', '_') + '_auroral_inds.h5')) onsets_borovsky = borovsky_id_algorithm(auroral_inds['AL']) onsets_borovsky = [ datetime(2005, 1, 1, tzinfo=UTC) + timedelta(minutes=m) for m in onsets_borovsky ] onset_lists['AL'] = get_tnums(onsets_borovsky, epoch) plasmoid_file = os.path.join( datadir, 'plasmoids_' + runprops['name'].replace('/', '').replace(' ', '_') + '.h5') if os.path.isfile(plasmoid_file): plasmoid_data = dm.fromHDF5(plasmoid_file) plasmoid_times = np.array([ datetime.strptime(s, '%Y-%m-%dT%H:%M:%S').replace(tzinfo=UTC) for s in plasmoid_data['time'] ]) plasmoid_times = plasmoid_times[plasmoid_data['x'] > -35] else: plasmoid_times = [] onset_lists['plasmoids'] = get_tnums(plasmoid_times, epoch) #midn_distances=(3,5,7,10,)#15,20,30,40,50) midn_distances = [7] dipolarizations = list( itertools.chain.from_iterable([ get_dipolarizations(runprops['name'], satname, datadir) for satname in ['goes10', 'goes12'] + ['midn_{0:02d}'.format(distance) for distance in midn_distances] ])) dipolarizations.sort() onset_lists['dipolarizations'] = get_tnums(dipolarizations, epoch) namestr = runprops['name'].replace('/', '').replace(' ', '_') mpb_onset_file = glob( os.path.join( datadir, 'John Haiducek - ' + namestr + '_mag_grid_lat=4?.matonset_tmax.txt')) if len(mpb_onset_file) == 1: onset, tmax = parse_onset_tmax(mpb_onset_file[0]) else: onset = [] onset_lists['MPB'] = get_tnums(onset, epoch) return onset_lists
def test_HDF5roundtripGZIP(self): """Data can go to hdf and back with compression""" dm.toHDF5(self.testfile, self.SDobj, compression='gzip') newobj = dm.fromHDF5(self.testfile) self.assertEqual(self.SDobj.attrs['global'], newobj.attrs['global']) np.testing.assert_almost_equal(self.SDobj['var'], newobj['var']) self.assertEqual(self.SDobj['var'].attrs['a'], newobj['var'].attrs['a']) dm.toHDF5(self.testfile, self.SDobj, mode='a', compression='gzip') newobj = dm.fromHDF5(self.testfile) self.assertEqual(self.SDobj.attrs['global'], newobj.attrs['global']) np.testing.assert_almost_equal(self.SDobj['var'], newobj['var']) self.assertEqual(self.SDobj['var'].attrs['a'], newobj['var'].attrs['a'])
def test_HDF5roundtrip_method(self): """Data can go to hdf and back""" self.SDobj.toHDF5(self.testfile) newobj = dm.fromHDF5(self.testfile) self.assertEqual(self.SDobj.attrs['global'], newobj.attrs['global']) np.testing.assert_almost_equal(self.SDobj['var'], newobj['var']) self.assertEqual(self.SDobj['var'].attrs['a'], newobj['var'].attrs['a']) dm.toHDF5(self.testfile, self.SDobj, mode='a') newobj = dm.fromHDF5(self.testfile) self.assertEqual(self.SDobj.attrs['global'], newobj.attrs['global']) np.testing.assert_almost_equal(self.SDobj['var'], newobj['var']) self.assertEqual(self.SDobj['var'].attrs['a'], newobj['var'].attrs['a'])
def test_toHDF5ListString(self): """Convert to HDF5, including a list of string in attributes""" a = dm.SpaceData() a.attrs['foo'] = ['hi'] dm.toHDF5(self.testfile, a, mode='a') newobj = dm.fromHDF5(self.testfile) self.assertEqual(a.attrs['foo'], newobj.attrs['foo'])
def test_copy_toHDF(self): """Removed code in #404 for an old python bug, issue4380, make sure it works""" a = dm.SpaceData({'dat': dm.dmarray([1, 2, 3])}) a2 = copy.deepcopy(a) a2.toHDF5(self.testfile, mode='a') newobj = dm.fromHDF5(self.testfile) np.testing.assert_array_equal([1, 2, 3], newobj['dat'])
def get_dipolarizations(run_name, satname, datadir='.'): namestr = run_name.replace('/', '').replace(' ', '_') satdata = dm.fromHDF5( os.path.join(datadir, namestr + '_' + satname + '.h5')) times = np.array(num2date(satdata['numtime'])) mlt = (np.arctan2(-satdata['Y'], -satdata['X']) * 12 / np.pi) % 24 newtimes = np.array([ datetime(2005, 1, 1, tzinfo=UTC) + timedelta(seconds=s * 60) for s in range(1440 * 31) ]) bx = interp_timeseries(satdata['bx'], times, newtimes) by = interp_timeseries(satdata['by'], times, newtimes) bz = interp_timeseries(satdata['bz'], times, newtimes) x = interp_timeseries(satdata['X'], times, newtimes) y = interp_timeseries(satdata['Y'], times, newtimes) mlt = interp_timeseries(mlt, times, newtimes) times = newtimes br = (bx * x + by * y) / np.sqrt(x**2 + y**2) theta = np.arctan2(bz, np.sqrt(bx**2 + by**2)) * 180 / np.pi dip_inds = find_dipolarizations_br_bz_theta(newtimes, br, bz, theta) night_inds = np.where((mlt < 6) | (mlt > 18))[0] dip_inds = np.intersect1d(dip_inds, night_inds) dip_times = times[dip_inds] satdata['time'] = newtimes satdata['bx'] = bx satdata['by'] = by satdata['bz'] = bz satdata['br'] = br satdata['theta'] = theta return dip_times
def data(orbit_no, show_avail, show_tree): ''' Gets the HEPD and MEPD data of the corresponding orbit number in the data path in SpacePy datamodel format. Also capable of showing available files and the selected file's file structure tree if show_avail or show_tree is set to True, respectively. ''' files = get_files(DATA_PATH, show_avail) HEPD_file_path, MEPD_file_path = get_file_paths(orbit_no, DATA_PATH, files) print('Selected orbit number: ', str(orbit_no), '\n') # Create SpacePy datamodel. HEPD_data = dm.fromHDF5(HEPD_file_path) MEPD_data = dm.fromHDF5(MEPD_file_path) if show_tree == True: file_tree(HEPD_data, MEPD_data) HEPD = isssData(orbit_no, HEPD_data, GROUP1, DATASET1, DATASET2) MEPD = isssData(orbit_no, MEPD_data, GROUP2, DATASET1, DATASET2) return HEPD, MEPD
def plot_sea_onset_comparison(run_name, var, ax, signature_types=signature_types): if run_name == 'obs': seadata['MPB'] = ['MPB ($nT^4$)', obs_mpb_v, obs_mpb_t] seadata['al'] = ['AL (nT)', sml, supermag_times] ylabel, data, times = seadata[var] artists, labels = plot_onset_sea(obs_signatures, obs_threshold, data, times, ylabel, ax, signature_types=signature_types) else: names = [runprops['name'] for runprops in run_properties] runprops = run_properties[names.index(run_name)] run_signatures = get_model_signature_lists(runprops, datadir=datadir) from spacepy import datamodel as dm auroral_inds = dm.fromHDF5( os.path.join( datadir, runprops['name'].replace('/', '').replace(' ', '_') + '_auroral_inds.h5')) al_time = [ datetime(2005, 1, 1, tzinfo=UTC) + timedelta(seconds=60 * m) for m in range(0, 1440 * 31) ] #try: # mpbdata=loadmat(os.path.join(datadir,'John Haiducek - '+runprops['name'].replace('/','').replace(' ','_')+'_mag_grid_lat=33_mpb.mat')) #except: # raise #mpb_t=[datetime(2005,1,1,tzinfo=UTC)+timedelta(seconds=m*60) for m in range(0,31*24*60)] #mpb_v=mpbdata['mpb'] mpb_t, mpb_v = parse_index(os.path.join(datadir, 'mpb_index.txt')) seadata['MPB'] = ['MPB ($nT^4$)', mpb_v, mpb_t] seadata['al'] = ['AL (nT)', auroral_inds['AL'], al_time] ylabel, data, times = seadata[var] artists, labels = plot_onset_sea(run_signatures, model_threshold, data, times, ylabel, ax, signature_types=signature_types) return artists, labels
def test_HDF5roundtrip2GZIP(self): """Data can go to hdf without altering datetimes in the datamodel with compression""" a = dm.SpaceData() a['foo'] = dm.SpaceData() dm.toHDF5(self.testfile, a, compression='gzip') newobj = dm.fromHDF5(self.testfile) self.assertEqual(a['foo'], newobj['foo']) a['bar'] = dm.dmarray([datetime.datetime(2000, 1, 1)]) dm.toHDF5(self.testfile, a, compression='gzip') self.assertEqual(a['bar'], dm.dmarray([datetime.datetime(2000, 1, 1)]))
def fromHDF5(cls, filepath, **kwargs): ''' Create a DataModel from a HDF5 data file. Attributes are not available without SpacePy. ''' if USE_SPACEPY: return cls.fromSpaceData(spdm.fromHDF5(filepath, **kwargs)) else: import h5py data_model = cls() path = '/' if 'path' in kwargs: path = kwargs['path'] h5file = h5py.File(filepath, mode='r') for key in h5file[path]: data_model[key] = numpy.array(h5file[path][key]) h5file.close() return data_model
@cache_result() def load_omni(dtstart, dtend, proxy=None): return get_cdf('sp_phys', 'OMNI_HRO_1MIN', dtstart, dtend, [ 'BX_GSE', 'BY_GSM', 'BZ_GSM', 'Vx', 'Vy', 'Vz', 'T', 'proton_density' ], proxy=proxy) # Read advect1d output advect1d_data = dm.fromHDF5('advected.h5') advect1d_data['time'] = advect1d_data['time'].astype('datetime64') # Retrieve start/end times of data and rount to nearest second starttime = advect1d_data['time'][0].astype('datetime64[s]').astype(datetime) endtime = advect1d_data['time'][-1].astype('datetime64[s]').astype(datetime) args = parse_args(starttime=starttime, endtime=endtime) starttime = args.start_time endtime = args.end_time source = args.source proxy = args.proxy # Fetch OMNI data omnidata = load_omni(starttime, endtime, proxy=proxy)
def test_toHDF5_method(self): """Convert to HDF5, using the method, catching #404""" a = dm.SpaceData({'dat': dm.dmarray([1, 2, 3])}) a.toHDF5(self.testfile, mode='a') newobj = dm.fromHDF5(self.testfile) np.testing.assert_array_equal([1, 2, 3], newobj['dat'])
logfiles = [] for filename in filenames: logfiles.append(SwmfLogFile(filename)) return LogSequence(logfiles) if __name__ == '__main__': cachefile = 'dipolarization_test.h5' import os from spacepy import datamodel as dm from matplotlib.dates import date2num, num2date if os.path.isfile(cachefile): satdata = dm.fromHDF5(cachefile) else: satfiles = build_log_sequence( '/data2/jhaiduce/substorms_Jan2005_young-comp/day*/GM/IO2/sat_goes10_20050101_n*.sat' ) satdata = dm.SpaceData() satdata['bz'] = dm.dmarray(satfiles['bz']) satdata['time'] = dm.dmarray(date2num(satfiles['time'])) satdata.toHDF5(cachefile) from matplotlib import pyplot as plt dt = (satdata['time'][1:] - satdata['time'][:-1]) * 3600 * 24 satdata['time'] = np.array(num2date(satdata['time']))