def FIRE_HiRes_L1_L2(datafile, ephemfile): full_data = dm.readJSONheadedASCII(datafile) ephem = dm.readJSONheadedASCII(ephemfile) data = Trim_data_file(full_data, ephem) labels = ephem.keys() ephem_fields = ['Lsimple', 'CDMAG_MLT'] dt = spt.Ticktock(data['Epoch']).TAI et = spt.Ticktock(ephem['DateTime']).TAI for i in range(len(ephem_fields)): print ephem_fields[i] y = ephem[ephem_fields[i]] nx = tb.interpol(dt, et, y) data[ephem_fields[i]] = dm.dmarray(nx) ephem_lat = ephem['Rgeod_LatLon'][:,0] ephem_lon = ephem['Rgeod_LatLon'][:,1] nx = tb.interpol(dt, et, ephem_lat) data['Lat'] = dm.dmarray(nx) nx = tb.interpol(dt, et, ephem_lon) data['Lon'] = dm.dmarray(nx) n_lines = len(data['Epoch']) eflux = np.zeros(n_lines,12) day = ephem['DateTime'][0][0:10] outfile = datafile[:-23] + day + '-HiRes_L2.txt' dm.toJSONheadedASCII(outfile, data) return data
def test_readJSONheadedASCII(self): """readJSONheadedASCII should read the test file""" dat = dm.readJSONheadedASCII(self.filename) keys = ['PerigeePosGeod', 'S_sc_to_pfn', 'S_pfs_to_Bmin', 'Pfs_gsm', 'Pfn_ED_MLAT', 'ED_R', 'Dst', 'DateTime', 'DOY', 'ED_MLON', 'IntModel', 'ApogeePosGeod', 'CD_MLON', 'S_sc_to_pfs', 'GpsTime', 'JulianDate', 'M_ref', 'ED_MLT', 'Pfs_ED_MLAT', 'Bfs_geo', 'Bm', 'Pfn_CD_MLON', 'CD_MLAT', 'Pfs_geo', 'Rsm', 'Pmin_gsm', 'Rgei', 'Rgsm', 'Pfs_CD_MLAT', 'S_total', 'Rgeod_Height', 'Date', 'Alpha', 'M_igrf', 'Pfs_CD_MLT', 'ED_MLAT', 'CD_R', 'PerigeeTimes', 'UTC', 'Pfn_ED_MLT', 'BoverBeq', 'Lsimple', 'Lstar', 'I', 'DipoleTiltAngle', 'K', 'Bmin_gsm', 'S_Bmin_to_sc', 'Bfs_gsm', 'L', 'ApogeeTimes', 'ExtModel', 'Kp', 'Pfs_geod_LatLon', 'MlatFromBoverBeq', 'Pfn_gsm', 'Loss_Cone_Alpha_n', 'Bfn_geo', 'Pfn_CD_MLAT', 'Rgeod_LatLon', 'Pfs_ED_MLT', 'Pfs_CD_MLON', 'Bsc_gsm', 'Pfn_geod_Height', 'Lm_eq', 'Rgse', 'Pfn_geod_LatLon', 'CD_MLT', 'FieldLineType', 'Pfn_CD_MLT', 'Pfs_geod_Height', 'Rgeo', 'InvLat_eq', 'M_used', 'Loss_Cone_Alpha_s', 'Bfn_gsm', 'Pfn_ED_MLON', 'Pfn_geo', 'InvLat', 'Pfs_ED_MLON'] if str is bytes: keys = [unicode(k) for k in keys] # make sure data has all the keys and no more or less for k in dat: self.assertTrue(k in keys) ind = keys.index(k) del keys[ind] self.assertEqual(len(keys), 0) dat = dm.readJSONheadedASCII(self.filename, convert=True) np.testing.assert_array_equal(dat['DateTime'], [datetime.datetime(2013, 2, 18, 0, 0), datetime.datetime(2013, 2, 18, 0, 5)])
def test_toJSONheadedASCII(self): """Write known datamodel to JSON-headed ASCII and ensure it has right stuff added""" a = dm.SpaceData() a.attrs['Global'] = 'A global attribute' a['Var1'] = dm.dmarray([1, 2, 3, 4, 5], attrs={'Local1': 'A local attribute'}) a['Var2'] = dm.dmarray([[8, 9], [9, 1], [3, 4], [8, 9], [7, 8]]) a['MVar'] = dm.dmarray([7.8], attrs={'Note': 'Metadata'}) t_file = tempfile.NamedTemporaryFile(delete=False) t_file.close() dm.toJSONheadedASCII(t_file.name, a, depend0='Var1', order=['Var1', 'Var2']) dat2 = dm.readJSONheadedASCII(t_file.name) #test global attr self.assertTrue(a.attrs == dat2.attrs) #test that metadata is back and all original keys are present for key in a['MVar'].attrs: self.assertTrue(key in dat2['MVar'].attrs) np.testing.assert_array_equal(a['MVar'], dat2['MVar']) #test vars are right np.testing.assert_almost_equal(a['Var1'], dat2['Var1']) np.testing.assert_almost_equal(a['Var2'], dat2['Var2']) #test for added dimension and start col self.assertTrue(dat2['Var1'].attrs['DIMENSION'] == [1]) self.assertTrue(dat2['Var2'].attrs['DIMENSION'] == [2]) os.remove(t_file.name)
def test_toHTML(self): """toHTML should give known output""" t_file = tempfile.NamedTemporaryFile(delete=False) t_file.close() dat = dm.readJSONheadedASCII(self.filename) dm.toHTML(t_file.name, dat, attrs=['DESCRIPTION', 'UNITS', 'ELEMENT_LABELS'], varLinks=True) if sys.platform == 'win32': #Different line endings expected = 12916 if str is bytes else 12892 else: expected = 12834 if str is bytes else 12810 #no u on unicode strings self.assertEqual(expected, os.path.getsize(t_file.name)) # not the best test but I am lazy os.remove(t_file.name)
def test_writeJSONMetadata(self): """reading metadata should give same keys as original datamodel""" dat = dm.readJSONMetadata(self.filename) # make sure data has all te keys and no more or less t_file = tempfile.NamedTemporaryFile(delete=False) t_file.close() dm.writeJSONMetadata(t_file.name, dat) dat2 = dm.readJSONheadedASCII(t_file.name) os.remove(t_file.name) keylist1 = sorted(dat.keys()) keylist2 = sorted(dat2.keys()) self.assertTrue(keylist1==keylist2) #now test that values in some metadata are identical self.assertTrue((dat['PerigeePosGeod'] == dat2['PerigeePosGeod']).all())
def FIRE_Context_L1_L2(datafile, ephemfile): full_data = dm.readJSONheadedASCII(datafile) ephem = dm.readJSONheadedASCII(ephemfile) meta = dm.readJSONheadedASCII(ephemfile) data = Trim_data_file(full_data, ephem) labels = ephem.keys() ephem_fields = test_ephem_list dt = spt.Ticktock(data['Epoch']).TAI et = spt.Ticktock(ephem['DateTime']).TAI for i in range(len(ephem_fields)): dim = np.size(ephem[ephem_fields[i]][0]) print ephem_fields[i], dim nx = np.empty([len(dt),dim]) if dim > 1: for j in range(dim): y = ephem[ephem_fields[i]][:,j] nx[:,j] = tb.interpol(dt, et, y) data[ephem_fields[i]] = dm.dmarray(nx, attrs = meta[ephem_fields[i]].attrs) else: y = ephem[ephem_fields[i]] nx = tb.interpol(dt, et, y) data[ephem_fields[i]] = dm.dmarray(nx, attrs = meta[ephem_fields[i]].attrs) col = deepcopy(data['Context'][:,0]) sur = deepcopy(data['Context'][:,1]) despike(col, 250, 10) despike(sur, 250, 10) col = (col/(6*9))*1033 sur = (sur/(6*23))*772 data['col'] = dm.dmarray(col, attrs = {'Description': 'Collimated Detector Energy Flux', 'SCALE_TYPE': 'log'}) data['sur'] = dm.dmarray(sur, attrs = {'Description': 'Surface Detector Energy Flux', 'SCALE_TYPE': 'log'}) day = ephem['DateTime'][0][0:10] outfile = datafile[:-25] + day + '-Context_L2.txt' order = ['Epoch', 'col', 'sur', 'Context'] # order = ['Epoch', 'Context'] order.extend(ephem_fields) dm.toJSONheadedASCII(outfile, data) return data
def test_toHTML(self): """toHTML should give known output""" t_file = tempfile.NamedTemporaryFile(delete=False) t_file.close() dat = dm.readJSONheadedASCII(self.filename) dm.toHTML(t_file.name, dat, attrs=['DESCRIPTION', 'UNITS', 'ELEMENT_LABELS'], varLinks=True) if sys.platform == 'win32': expected = 12916 #different line-endings else: if str is bytes: expected = 12834 else: expected = 12810 #no u on the unicode strings self.assertEqual(expected, os.path.getsize(t_file.name)) # not the best test but I am lazy os.remove(t_file.name)
def readfiles(files): """ read in the files and store the data we need to data """ global data # glabal means we can edit this files = set(files) # make this a unique set of files so that the same is not read twice for f in files: try: dat = dm.readJSONheadedASCII(f) except (IOError, ValueError): print('File {0} is not valid JSONheadedascii, skipped'.format(f)) continue if not isHires(dat) and not isConfig(dat): print('File {0} is not hires or config, skipped'.format(f)) continue # not the kind of file we want ## file is now for sure wanted store its info to data data[f] = dat
def parseData_Times(fname): """ TODO in here if we dont get at least a page of time for a data type is likely missing, so probably don't ask TODO if the data times duration is less than 90 minutes then something happened making the data in that sepment suspect """ if fname is None: return None data = dm.readJSONheadedASCII(fname, convert={'Epoch':lambda x: dup.parse(x, ignoretz=True), 'Time':lambda x: dup.parse(x, ignoretz=True)}) t1 = np.asarray([v.replace(microsecond=0) for v in data['Epoch']]) t2 = np.asarray([v.replace(microsecond=0) for v in data['Time']]) outdat = dm.SpaceData() outdat['On'] = np.asarray( [t1[data['Mode'].astype(np.bool)], t2[data['Mode'].astype(np.bool)]]).T outdat['Off'] = np.asarray( [t1[~data['Mode'].astype(np.bool)], t2[~data['Mode'].astype(np.bool)]]).T return outdat
def test_toJSONheadedASCII_method(self): """Write known datamodel to JSON-headed ASCII and ensure it has right stuff added""" a = dm.SpaceData() a.attrs['Global'] = 'A global attribute' a['Var1'] = dm.dmarray([1,2,3,4,5], attrs={'Local1': 'A local attribute'}) a['Var2'] = dm.dmarray([[8,9],[9,1],[3,4],[8,9],[7,8]]) a['MVar'] = dm.dmarray([7.8], attrs={'Note': 'Metadata'}) t_file = tempfile.NamedTemporaryFile(delete=False) t_file.close() a.toJSONheadedASCII(t_file.name, depend0='Var1', order=['Var1','Var2']) dat2 = dm.readJSONheadedASCII(t_file.name) #test global attr self.assertTrue(a.attrs==dat2.attrs) #test that metadata is back and all original keys are present for key in a['MVar'].attrs: self.assertTrue(key in dat2['MVar'].attrs) np.testing.assert_array_equal(a['MVar'], dat2['MVar']) #test vars are right np.testing.assert_almost_equal(a['Var1'], dat2['Var1']) np.testing.assert_almost_equal(a['Var2'], dat2['Var2']) #test for added dimension and start col self.assertTrue(dat2['Var1'].attrs['DIMENSION']==[1]) self.assertTrue(dat2['Var2'].attrs['DIMENSION']==[2]) os.remove(t_file.name)
if not os.path.isfile(f): parser.error("File {0} does not exist".format(f)) #============================================================================== # deal with the filetype options #============================================================================== for f in infiles: try: tp = determineFileType(f) except (ValueError, NotImplementedError): # could not determine the type, die parser.error("Could not determine the file type: {0}".format(f)) dat = dm.readJSONheadedASCII(f) dat['Epoch'][...] = [dup.parse(v) for v in dat['Epoch'][...]] fig = plt.figure() ax = fig.add_subplot(111) imname = f + '_ql_{0}.png'.format(datetime.datetime.now()).replace(' ', 'T').replace(':', '-') if tp == 'configfile': for var in range(16): ax.plot(dat['Epoch'], dat['reg{0:02}'.format(var)], label='reg{0:02}'.format(var)) ax.set_ylim((-10, 300)) ax.set_xlim((ax.get_xlim()[0], ax.get_xlim()[1]+(ax.get_xlim()[1]-ax.get_xlim()[0])*0.45)) ax.legend(loc='upper right') fig.savefig(imname) plt.close() elif tp == 'mbpfile':
try: date1=datetime.datetime.strftime(dt1, '%Y%m%d') print(date1) # to wake up the nfs server I hope os.chdir('/Users/loisks/Desktop/liemohn10/loisks/'+dir[idir]) emf_file=name[idir]+'_WFR-spectral-matrix_emfisis-L2_'+date1+'*.cdf' gemf=glob.glob(emf_file) pyf=pycdf.CDF(gemf[0]) os.chdir('..') # get the polarization polarization=gM.mag_SVD(pyf, ifreq) # # get the emphermeris file os.chdir('/Users/loisks/Desktop/liemohn10/loisks/EMPHEMERIS_'+sat[idir]) hope_emphem=glob.glob('rbsp'+lsat[idir]+'_def_MagEphem_OP77Q_'+date1+'*.txt')[0] pyf2=dm.readJSONheadedASCII(hope_emphem) L_emphem=np.nanmedian(pyf2['L'], axis=1) # L from emphemeris L_emphem[L_emphem<0]=np.nan MLT_emphem=pyf2['CDMAG_MLT'] # MLT from ephemeris MLT_emphem[MLT_emphem<0]=np.nan epoch_ephem=epochL4=pd.DatetimeIndex(pyf2['DateTime']) # # great, now resample this stuff LEMP=pd.DataFrame({'L':L_emphem},index=epoch_ephem) MLTEMP=pd.DataFrame({'MLT':MLT_emphem},index=epoch_ephem) # get the coordinates from the EMFISIS L4 files # time array is 14400, reduce by factor of 10 Kp=np.array(KpArr[date1])
def get_omni(ticks, dbase='QDhourly', **kwargs): ''' Returns Qin-Denton OMNI values, interpolated to any time-base from a default hourly resolution The update function in toolbox retrieves all available hourly Qin-Denton data, and this function accesses that and interpolates to the given times, returning the OMNI values as a SpaceData (dict-like) with Kp, Dst, dens, velo, Pdyn, ByIMF, BzIMF, G1, G2, G3, etc. (see also http://www.dartmouth.edu/~rdenton/magpar/index.html and http://www.agu.org/pubs/crossref/2007/2006SW000296.shtml ) Parameters ========== ticks : Ticktock class or array-like of datetimes time values for desired output dbase : str (optional) Select data source, options are 'QDhourly', 'OMNI2hourly', 'Mergedhourly' Note - Custom data sources can be specified in the spacepy config file as described in the module documentation. Returns ======= out : spacepy.datamodel.SpaceData containing all Qin-Denton values at times given by ticks Examples ======== >>> import spacepy.time as spt >>> import spacepy.omni as om >>> ticks = spt.Ticktock(['2002-02-02T12:00:00', '2002-02-02T12:10:00'], 'ISO') >>> d = om.get_omni(ticks) >>> d.tree(levels=1) + |____ByIMF |____Bz1 |____Bz2 |____Bz3 |____Bz4 |____Bz5 |____Bz6 |____BzIMF |____DOY |____Dst |____G1 |____G2 |____G3 |____Hr |____Kp |____Pdyn |____Qbits |____RDT |____UTC |____W1 |____W2 |____W3 |____W4 |____W5 |____W6 |____Year |____akp3 |____dens |____ticks |____velo Notes ===== Note about Qbits: If the status variable is 2, the quantity you are using is fairly well determined. If it is 1, the value has some connection to measured values, but is not directly measured. These values are still better than just using an average value, but not as good as those with the status variable equal to 2. If the status variable is 0, the quantity is based on average quantities, and the values listed are no better than an average value. The lower the status variable, the less confident you should be in the value. ''' dbase_options = { 'QDhourly': 1, 'OMNI2hourly': 2, 'Mergedhourly': 3, 'Test': -9, } if not isinstance(ticks, spt.Ticktock): try: ticks = spt.Ticktock(ticks, 'UTC') except: raise TypeError( 'get_omni: Input times must be a Ticktock object or a list of datetime objects' ) if not dbase in dbase_options: from spacepy import config if dbase in config: #If a dbase is specified that isn't a default, then it MUST be in the spacepy config qdpath = os.path.split(os.path.split(config[dbase])[0])[0] if not os.path.isdir(qdpath): raise IOError( 'Specified dbase ({0}) does not have a valid location ({1})' .format(dbase, config[dbase])) days = list(set([tt.date() for tt in ticks.UTC])) flist = [''] * len(days) fnpath, fnformat = os.path.split(config[dbase]) for idx, day in enumerate(days): dp = fnpath.replace('YYYY', '{0}'.format(day.year)) df = fnformat.replace('YYYY', '{0}'.format(day.year)) df = df.replace('MM', '{0:02d}'.format(day.month)) df = df.replace('DD', '{0:02d}'.format(day.day)) flist[idx] = os.path.join(dp, df) if 'convert' in kwargs: convdict = kwargs['convert'] else: convdict = True #set to True as default? if 'interp' not in kwargs: kwargs['interp'] = True data = readJSONheadedASCII(sorted(flist), convert=convdict) omniout = SpaceData() time_var = [ var for var in ['DateTime', 'Time', 'Epoch', 'UTC'] if var in data ] if time_var: use_t_var = time_var[0] else: #no obvious time variable in input files ... can't continue raise ValueError('No clear time variable in file') if kwargs['interp'] is True: data['RDT'] = spt.Ticktock(data[use_t_var]).RDT keylist = sorted(data.keys()) dum = keylist.pop(keylist.index(use_t_var)) for key in keylist: try: omniout[key] = dmarray( np.interp(ticks.RDT, data['RDT'], data[key], left=np.NaN, right=np.NaN)) omniout[key].attrs = dmcopy(data[key].attrs) except: try: omniout[key] = dmfilled( [len(ticks.RDT), data[key].shape[1]], fillval=np.NaN, attrs=dmcopy(data[key].attrs)) for col in range(data[key].shape[1]): omniout[key][:, col] = np.interp(ticks.RDT, data['RDT'], data[key][:, col], left=np.NaN, right=np.NaN) except ValueError: print( 'Failed to interpolate {0} to new time base, skipping variable' .format(key)) except IndexError: print( 'Variable {0} appears to be non-record varying, skipping interpolation' .format(key)) omniout[key] = data[key] omniout['UTC'] = ticks.UTC else: #Trim to specified times inds = tOverlapHalf([ticks[0].RDT, ticks[-1].RDT], spt.Ticktock(data['DateTime']).RDT) for key in data: if len(inds) == len(data[key]): omniout[key] = data[key][inds] else: #is ancillary data omniout[key] = data[key] #TODO: convert to same format as OMNI/QD read (or vice versa) omniout['UTC'] = omniout[use_t_var] return omniout else: raise IOError( 'Specified dbase ({0}) must be specified in spacepy.config'. format(dbase)) def getattrs(hf, key): out = {} if hasattr(hf[key], 'attrs'): for kk, value in hf[key].attrs.items(): try: out[kk] = value except: pass return out def HrFromDT(indt): hour = indt.hour minute = indt.minute second = indt.second musecond = indt.microsecond return hour + (minute / 60.0) + (second / 3600.0) + (musecond / 3600.0e3) import h5py as h5 fname, QDkeylist, O2keylist = '', [], [] omnivals = SpaceData() dbase_select = dbase_options[dbase] if dbase_select in [1, 3, -9]: if dbase_select > 0: ldb = 'QDhourly' fln = omnifln else: ldb = 'Test' fln = testfln with h5.File(fln, 'r') as hfile: QDkeylist = [kk for kk in hfile if kk not in ['Qbits', 'UTC']] st, en = ticks[0].RDT, ticks[-1].RDT ##check that requested requested times are within range of data enval, stval = omnirange(dbase=ldb)[1], omnirange(dbase=ldb)[0] if (ticks.UTC[0] > enval) or (ticks[-1] < stval): raise ValueError('Requested dates are outside data range') if (ticks.UTC[-1] > enval) or (ticks[0] < stval): print( 'Warning: Some requested dates are outside data range ({0})' .format(ldb)) inds = tOverlapHalf([st, en], hfile['RDT'], presort=True) #returns an xrange inds = indsFromXrange(inds) if inds[0] < 1: inds[0] = 1 sl_op = slice(inds[0] - 1, inds[-1] + 2) fname = ','.join([fname, hfile.filename]) omnivals.attrs = getattrs(hfile, '/') for key in QDkeylist: omnivals[key] = dmarray( hfile[key][sl_op]) #TODO: add attrs from h5 omnivals[key].attrs = getattrs(hfile, key) for key in hfile['Qbits']: omnivals['Qbits<--{0}'.format(key)] = dmarray( hfile['/Qbits/{0}'.format(key)][sl_op]) omnivals['Qbits<--{0}'.format(key)].attrs = getattrs( hfile, '/Qbits/{0}'.format(key)) QDkeylist.append('Qbits<--{0}'.format(key)) if dbase_options[dbase] == 2 or dbase_options[dbase] == 3: ldb = 'OMNI2hourly' with h5.File(omni2fln) as hfile: O2keylist = [kk for kk in hfile if kk not in ['Epoch', 'RDT']] st, en = ticks[0].RDT, ticks[-1].RDT ##check that requested requested times are within range of data enval, stval = omnirange(dbase=ldb)[1], omnirange(dbase=ldb)[0] if (ticks[0].UTC > enval) or (ticks[-1] < stval): raise ValueError('Requested dates are outside data range') if (ticks[-1].UTC > enval) or (ticks[0] < stval): print( 'Warning: Some requested dates are outside data range ({0})' .format(ldb)) inds = tOverlapHalf([st, en], hfile['RDT'], presort=True) #returns an xrange inds = indsFromXrange(inds) if inds[0] < 1: inds[0] = 1 sl_op = slice(inds[0] - 1, inds[-1] + 2) fname = ','.join([fname, hfile.filename]) omnivals.attrs = getattrs( hfile, '/' ) #TODO: This overwrites the previous set on Merged load... Fix! omnivals['RDT_OMNI'] = dmarray(hfile['RDT'][sl_op]) for key in O2keylist: omnivals[key] = dmarray( hfile[key][sl_op]) #TODO: add attrs from h5 omnivals[key].attrs = getattrs(hfile, key) if dbase_options[dbase] == 3: #prune "merged" SpaceData sigmas = [key for key in omnivals if 'sigma' in key] for sk in sigmas: del omnivals[sk] bees = [key for key in omnivals if re.search('B._', key)] for bs in bees: del omnivals[bs] aves = [key for key in omnivals if ('_ave' in key) or ('ave_' in key)] for av in aves: del omnivals[av] omniout = SpaceData(attrs=dmcopy(omnivals.attrs)) omniout.attrs['filename'] = fname[1:] ###print('QDkeys: {0}\n\nO2keys: {1}'.format(QDkeylist, O2keylist)) for key in sorted(omnivals.keys()): if key in O2keylist: omniout[key] = dmarray( np.interp(ticks.RDT, omnivals['RDT_OMNI'], omnivals[key], left=np.NaN, right=np.NaN)) #set metadata -- assume this has been set properly in d/l'd file to match ECT-SOC files omniout[key].attrs = dmcopy(omnivals[key].attrs) elif key in QDkeylist: omniout[key] = dmarray( np.interp(ticks.RDT, omnivals['RDT'], omnivals[key], left=np.NaN, right=np.NaN)) omniout[key].attrs = dmcopy(omnivals[key].attrs) if key == 'G3': #then we have all the Gs omniout['G'] = dmarray( np.vstack([omniout['G1'], omniout['G2'], omniout['G3']]).T) omniout['G'].attrs = dmcopy(omnivals['G1'].attrs) for i in range(1, 4): del omniout['G{0}'.format(i)] if key == 'W6': omniout['W'] = dmarray( np.vstack([ omniout['W1'], omniout['W2'], omniout['W3'], omniout['W4'], omniout['W5'], omniout['W6'] ]).T) omniout['W'].attrs = dmcopy(omnivals['W1'].attrs) for i in range(1, 7): del omniout['W{0}'.format(i)] if 'Qbits' in key: #Qbits are integer vals, higher is better, so floor to get best representation of interpolated val omniout[key] = np.floor(omnivals[key]) omniout[key].attrs = dmcopy(omnivals[key].attrs) if 'G3' in key: #then we have all the Gs omniout['Qbits<--G'] = dmarray( np.vstack([ omniout['Qbits<--G1'], omniout['Qbits<--G2'], omniout['Qbits<--G3'] ]).T) for i in range(1, 4): del omniout['Qbits<--G{0}'.format(i)] if 'W6' in key: omniout['Qbits<--W'] = dmarray( np.vstack([ omniout['Qbits<--W1'], omniout['Qbits<--W2'], omniout['Qbits<--W3'], omniout['Qbits<--W4'], omniout['Qbits<--W5'], omniout['Qbits<--W6'] ]).T) for i in range(1, 7): del omniout['Qbits<--W{0}'.format(i)] omniout['ticks'] = ticks omniout['UTC'] = ticks.UTC omniout['Hr'] = dmarray([HrFromDT(val) for val in omniout['UTC']]) omniout['Year'] = dmarray([val.year for val in omniout['UTC']]) omniout = unflatten(omniout) return omniout
date=datetime.datetime.strftime(dt1, '%Y%m%d') os.chdir('/Users/loisks/Desktop/liemohn10/loisks/EMFISIS_'+dir[idir]) emf_file='*'+date+'*.cdf' gemf=glob.glob(emf_file) pyf=pycdf.CDF(gemf[0]) bTime=pd.DatetimeIndex(pyf['Epoch'][...]) bMag=pyf['Magnitude'][...] dB=pd.DataFrame({'Bmag':bMag}, index=bTime) # # now have to get the emphermeris data os.chdir('..') os.chdir('EMPHEMERIS_'+dir[idir]) emp_file='*'+date+'*.txt' gemp=glob.glob(emp_file) pyfemp=dm.readJSONheadedASCII(gemp[0]) L_emphem=pyfemp['L'] # L from emphemeris L_emphem=np.nanmedian(L_emphem, axis=1) L=np.array(L_emphem) Kp_emphem=pyfemp['Kp'] # Kp index from ephemeris file MLAT_emphem=pyfemp['CDMAG_MLAT'] # MLAT from ephemeris MLAT_emphem[MLAT_emphem<-100]=np.nan MLAT=np.array(MLAT_emphem) MLT_emphem=pyfemp['CDMAG_MLT'] # MLAT from ephemeris MLT_emphem[MLT_emphem<-100]=np.nan MLT=np.array(MLT_emphem) # emphemeris data is every minute rt = pd.period_range(date,periods=1441, freq='T').to_timestamp() # # resample the B data B=np.array(dB['Bmag'].resample('1min', how='mean').reindex(index=rt,fill_value=np.nan))
skip # get the EFW data os.chdir('/Users/loisks/Desktop/liemohn10/loisks/EFW_L3_'+satellites[isat]) f=glob.glob('*'+iDate+'*') pyf=pycdf.CDF(f[0]) potential=-1*pyf['Vavg'][...] rng = pd.period_range(iDate,periods=1440, freq='T').to_timestamp() pEpoch=pd.DatetimeIndex(pyf['epoch'][...]) df=pd.DataFrame(potential, index=pEpoch, columns=['potential']) Phi=df['potential'].resample('1min', how='median').reindex(rng, fill_value=np.nan) # # now get the ephemeris data eclipseFlag=np.zeros(len(rng)) os.chdir('/Users/loisks/Desktop/liemohn10/loisks/EMPHEMERIS_'+satellites[isat]) files=glob.glob('*'+iDate+'*') pyfem=dm.readJSONheadedASCII(files[0]) gseCoords=np.swapaxes(pyfem["Rgse"], 1, 0) for iGSE in range(len(rng)): # subtract out an RE to get difference above the surface if ((gseCoords[1][iGSE])**2 + (gseCoords[2][iGSE])**2) < 1: if gseCoords[0][iGSE] < 0: # satellite is in Earth's shadow eclipseFlag[iGSE]=1 print 'eclipse flag' LShell=np.swapaxes(pyfem['L'],1,0)[0] # weird multiple rows with L MLT=pyfem['CDMAG_MLT'] # now let's get dates and times where there is charging and it's # not in eclipse keyTimes=np.where((np.array(Phi) < 0) & (eclipseFlag != 1))[0] chargingYa=np.array(Phi[keyTimes])
return metrics if __name__ == '__main__': #load output from month-long run infile = sys.argv[1] ##infile = 'log_n000000.log' data = ram.LogFile(infile) useBiot = True useDPS = False #get Sym-H from Kyoto WDC #kyotodata = kyo.fetch('sym', data['time'][0], data['time'][-1]+dt.timedelta(minutes=1)) #read Kyoto data from saved file kyotodata = dm.readJSONheadedASCII('kyotodata_Jan2005.txt') #make plot fig = plt.figure(figsize=(10, 5)) ax = fig.add_subplot(111) ax.plot(kyotodata['time'], kyotodata['sym-h'], color='black', label='Sym-H (Kyoto)') if useBiot and not useDPS: ax.plot(data['time'], data['dstBiot'], color='crimson', label='Sym-H (RAM)') elif useDPS and not useBiot: ax.plot(data['time'],
def test_toJSONheadedASCII_method_404(self): """Convert to toJSONheadedASCII, using the method, catching #404""" a = dm.SpaceData({'dat': dm.dmarray([1, 2, 3])}) a.toJSONheadedASCII(self.testfile, mode='a') newobj = dm.readJSONheadedASCII(self.testfile) np.testing.assert_array_equal([1, 2, 3], newobj['dat'])
def get_omni(ticks, dbase='QDhourly', **kwargs): ''' Returns Qin-Denton OMNI values, interpolated to any time-base from a default hourly resolution The update function in toolbox retrieves all available hourly Qin-Denton data, and this function accesses that and interpolates to the given times, returning the OMNI values as a SpaceData (dict-like) with Kp, Dst, dens, velo, Pdyn, ByIMF, BzIMF, G1, G2, G3, etc. (see also http://www.dartmouth.edu/~rdenton/magpar/index.html and http://www.agu.org/pubs/crossref/2007/2006SW000296.shtml ) Parameters ========== ticks : Ticktock class or array-like of datetimes time values for desired output dbase : str (optional) Select data source, options are 'QDhourly', 'OMNI2', 'Mergedhourly' Note - Custom data sources can be specified in the spacepy config file as described in the module documentation. Returns ======= out : spacepy.datamodel.SpaceData containing all Qin-Denton values at times given by ticks Examples ======== >>> import spacepy.time as spt >>> import spacepy.omni as om >>> ticks = spt.Ticktock(['2002-02-02T12:00:00', '2002-02-02T12:10:00'], 'ISO') >>> d = om.get_omni(ticks) >>> d.tree(levels=1) + |____ByIMF |____Bz1 |____Bz2 |____Bz3 |____Bz4 |____Bz5 |____Bz6 |____BzIMF |____DOY |____Dst |____G1 |____G2 |____G3 |____Hr |____Kp |____Pdyn |____Qbits |____RDT |____UTC |____W1 |____W2 |____W3 |____W4 |____W5 |____W6 |____Year |____akp3 |____dens |____ticks |____velo Notes ===== Note about Qbits: If the status variable is 2, the quantity you are using is fairly well determined. If it is 1, the value has some connection to measured values, but is not directly measured. These values are still better than just using an average value, but not as good as those with the status variable equal to 2. If the status variable is 0, the quantity is based on average quantities, and the values listed are no better than an average value. The lower the status variable, the less confident you should be in the value. ''' dbase_options = {'QDhourly' : 1, 'OMNI2hourly' : 2, 'Mergedhourly': 3, 'Test' : -9, } if not isinstance(ticks, spt.Ticktock): try: ticks = spt.Ticktock(ticks, 'UTC') except: raise TypeError('get_omni: Input times must be a Ticktock object or a list of datetime objects') if not dbase in dbase_options: from spacepy import config if dbase in config: #If a dbase is specified that isn't a default, then it MUST be in the spacepy config qdpath = os.path.split(os.path.split(config[dbase])[0])[0] if not os.path.isdir(qdpath): raise IOError('Specified dbase ({0}) does not have a valid location ({1})'.format(dbase, config[dbase])) days = list(set([tt.date() for tt in ticks.UTC])) flist = ['']*len(days) fnpath, fnformat = os.path.split(config[dbase]) for idx, day in enumerate(days): dp = fnpath.replace('YYYY', '{0}'.format(day.year)) df = fnformat.replace('YYYY', '{0}'.format(day.year)) df = df.replace('MM', '{0:02d}'.format(day.month)) df = df.replace('DD', '{0:02d}'.format(day.day)) flist[idx] = os.path.join(dp, df) if 'convert' in kwargs: convdict = kwargs['convert'] else: convdict = True #set to True as default? if 'interp' not in kwargs: kwargs['interp'] = True data = readJSONheadedASCII(sorted(flist), convert=convdict) omniout = SpaceData() time_var = [var for var in ['DateTime', 'Time', 'Epoch', 'UTC'] if var in data] if time_var: use_t_var = time_var[0] else: #no obvious time variable in input files ... can't continue raise ValueError('No clear time variable in file') if kwargs['interp'] is True: data['RDT'] = spt.Ticktock(data[use_t_var]).RDT keylist = sorted(data.keys()) dum = keylist.pop(keylist.index(use_t_var)) for key in keylist: try: omniout[key] = dmarray(np.interp(ticks.RDT, data['RDT'], data[key], left=np.NaN, right=np.NaN)) omniout[key].attrs = dmcopy(data[key].attrs) except: try: omniout[key] = dmfilled([len(ticks.RDT), data[key].shape[1]], fillval=np.NaN, attrs=dmcopy(data[key].attrs)) for col in range(data[key].shape[1]): omniout[key][:,col] = np.interp(ticks.RDT, data['RDT'], data[key][:,col], left=np.NaN, right=np.NaN) except ValueError: print('Failed to interpolate {0} to new time base, skipping variable'.format(key)) except IndexError: print('Variable {0} appears to be non-record varying, skipping interpolation'.format(key)) omniout[key] = data[key] omniout['UTC'] = ticks.UTC else: #Trim to specified times inds = tOverlapHalf([ticks[0].RDT, ticks[-1].RDT], spt.Ticktock(data['DateTime']).RDT) for key in data: if len(inds) == len(data[key]): omniout[key] = data[key][inds] else: #is ancillary data omniout[key] = data[key] #TODO: convert to same format as OMNI/QD read (or vice versa) omniout['UTC'] = omniout[use_t_var] return omniout else: raise IOError('Specified dbase ({0}) must be specified in spacepy.config'.format(dbase)) def getattrs(hf, key): out = {} if hasattr(hf[key],'attrs'): for kk, value in hf[key].attrs.items(): try: out[kk] = value except: pass return out def HrFromDT(indt): hour = indt.hour minute = indt.minute second = indt.second musecond = indt.microsecond return hour+(minute/60.0)+(second/3600.0)+(musecond/3600.0e3) import h5py as h5 fname, QDkeylist, O2keylist = '', [], [] omnivals = SpaceData() dbase_select = dbase_options[dbase] if dbase_select in [1, 3, -9]: if dbase_select > 0: ldb = 'QDhourly' fln = omnifln else: ldb = 'Test' fln = testfln with h5.File(fln, 'r') as hfile: QDkeylist = [kk for kk in hfile if kk not in ['Qbits', 'UTC']] st, en = ticks[0].RDT, ticks[-1].RDT ##check that requested requested times are within range of data enval, stval = omnirange(dbase=ldb)[1], omnirange(dbase=ldb)[0] if (ticks.UTC[0]>enval) or (ticks[-1]<stval): raise ValueError('Requested dates are outside data range') if (ticks.UTC[-1]>enval) or (ticks[0]<stval): print('Warning: Some requested dates are outside data range ({0})'.format(ldb)) inds = tOverlapHalf([st, en], hfile['RDT'], presort=True) #returns an xrange inds = indsFromXrange(inds) if inds[0] < 1: inds[0] = 1 sl_op = slice(inds[0]-1, inds[-1]+2) fname = ','.join([fname,hfile.filename]) omnivals.attrs = getattrs(hfile, '/') for key in QDkeylist: omnivals[key] = dmarray(hfile[key][sl_op]) #TODO: add attrs from h5 omnivals[key].attrs = getattrs(hfile, key) for key in hfile['Qbits']: omnivals['Qbits<--{0}'.format(key)] = dmarray(hfile['/Qbits/{0}'.format(key)][sl_op]) omnivals['Qbits<--{0}'.format(key)].attrs = getattrs(hfile, '/Qbits/{0}'.format(key)) QDkeylist.append('Qbits<--{0}'.format(key)) if dbase_options[dbase] == 2 or dbase_options[dbase] == 3: ldb = 'OMNI2hourly' with h5.File(omni2fln) as hfile: O2keylist = [kk for kk in hfile if kk not in ['Epoch','RDT']] st, en = ticks[0].RDT, ticks[-1].RDT ##check that requested requested times are within range of data enval, stval = omnirange(dbase=ldb)[1], omnirange(dbase=ldb)[0] if (ticks[0].UTC>enval) or (ticks[-1]<stval): raise ValueError('Requested dates are outside data range') if (ticks[-1].UTC>enval) or (ticks[0]<stval): print('Warning: Some requested dates are outside data range ({0})'.format(ldb)) inds = tOverlapHalf([st, en], hfile['RDT'], presort=True) #returns an xrange inds = indsFromXrange(inds) if inds[0] < 1: inds[0] = 1 sl_op = slice(inds[0]-1, inds[-1]+2) fname = ','.join([fname,hfile.filename]) omnivals.attrs = getattrs(hfile, '/') #TODO: This overwrites the previous set on Merged load... Fix! omnivals['RDT_OMNI'] = dmarray(hfile['RDT'][sl_op]) for key in O2keylist: omnivals[key] = dmarray(hfile[key][sl_op]) #TODO: add attrs from h5 omnivals[key].attrs = getattrs(hfile, key) if dbase_options[dbase] == 3: #prune "merged" SpaceData sigmas = [key for key in omnivals if 'sigma' in key] for sk in sigmas: del omnivals[sk] bees = [key for key in omnivals if re.search('B._', key)] for bs in bees: del omnivals[bs] aves = [key for key in omnivals if ('_ave' in key) or ('ave_' in key)] for av in aves: del omnivals[av] omniout = SpaceData(attrs=dmcopy(omnivals.attrs)) omniout.attrs['filename'] = fname[1:] ###print('QDkeys: {0}\n\nO2keys: {1}'.format(QDkeylist, O2keylist)) for key in sorted(omnivals.keys()): if key in O2keylist: omniout[key] = dmarray(np.interp(ticks.RDT, omnivals['RDT_OMNI'], omnivals[key], left=np.NaN, right=np.NaN)) #set metadata -- assume this has been set properly in d/l'd file to match ECT-SOC files omniout[key].attrs = dmcopy(omnivals[key].attrs) elif key in QDkeylist: omniout[key] = dmarray(np.interp(ticks.RDT, omnivals['RDT'], omnivals[key], left=np.NaN, right=np.NaN)) omniout[key].attrs = dmcopy(omnivals[key].attrs) if key == 'G3': #then we have all the Gs omniout['G'] = dmarray(np.vstack([omniout['G1'], omniout['G2'], omniout['G3']]).T) omniout['G'].attrs = dmcopy(omnivals['G1'].attrs) for i in range(1,4): del omniout['G{0}'.format(i)] if key == 'W6': omniout['W'] = dmarray(np.vstack([omniout['W1'], omniout['W2'], omniout['W3'], omniout['W4'], omniout['W5'], omniout['W6']]).T) omniout['W'].attrs = dmcopy(omnivals['W1'].attrs) for i in range(1,7): del omniout['W{0}'.format(i)] if 'Qbits' in key: #Qbits are integer vals, higher is better, so floor to get best representation of interpolated val omniout[key] = np.floor(omnivals[key]) omniout[key].attrs = dmcopy(omnivals[key].attrs) if 'G3' in key: #then we have all the Gs omniout['Qbits<--G'] = dmarray(np.vstack([omniout['Qbits<--G1'], omniout['Qbits<--G2'], omniout['Qbits<--G3']]).T) for i in range(1,4): del omniout['Qbits<--G{0}'.format(i)] if 'W6' in key: omniout['Qbits<--W'] = dmarray(np.vstack([omniout['Qbits<--W1'], omniout['Qbits<--W2'], omniout['Qbits<--W3'], omniout['Qbits<--W4'], omniout['Qbits<--W5'], omniout['Qbits<--W6']]).T) for i in range(1,7): del omniout['Qbits<--W{0}'.format(i)] omniout['ticks'] = ticks omniout['UTC'] = ticks.UTC omniout['Hr'] = dmarray([HrFromDT(val) for val in omniout['UTC']]) omniout['Year'] = dmarray([val.year for val in omniout['UTC']]) omniout = unflatten(omniout) return omniout
def getKondrashovSW( fn='../ref_data/1989/Kondrashov_SSA_19890312_19890314.txt'): data = dm.readJSONheadedASCII(fn, convert=True) return data
bcvals=coord.Coords([[GEOBx[igp],GEOBy[igp],GEOBz[igp]]], 'GEO', 'car') cvals.ticks=Ticktock([ctime], 'UTC') bcvals.ticks=Ticktock([ctime], 'UTC') new_coord=cvals.convert(desired_coords[ic],'car') newB_coord=bcvals.convert(desired_coords[ic], 'car') XGSE[igp]=new_coord.x; YGSE[igp]=new_coord.y; ZGSE[igp]=new_coord.z Bx[igp]=newB_coord.x; By[igp]=newB_coord.y; Bz[igp]=newB_coord.z # # now we have the interpolated pressures... get the Kp, L, and MLT # from HOPE Ephemeris # os.chdir('/Users/loisks/Desktop/liemohn10/loisks/EMPHEMERIS_'+sats[iSat]) f2=glob.glob('rbsp'+lsats[iSat]+'_def_MagEphem_OP77Q_'+date1+'*.txt')[0] pyf2=dm.readJSONheadedASCII(f2) # get params LShell=np.array(np.nanmean(np.array(pyf2['L']), axis=1)) # weird multiple rows with L MLT=np.array(pyf2['CDMAG_MLT']) Kp=np.array(KpArr[date1]) # in minutes # # now we sort and compare # 0.5 MLT, 0.25 L, and Kp =1 bins up to kp 9 for iKp in range(9): for iMLT in range(nMLT): for iL in range(nL): temp=np.where((MLT >= MLTbins[iMLT]-0.25) & (MLT < MLTbins[iMLT]+0.25))[0] temp2=np.where((LShell >= Lbins[iL]-.125) & (LShell < Lbins[iL]+0.125))[0] temp3=np.where(( Kp >= iKp) & ( Kp < iKp+1))[0] # get the set
XGSE[igp] = new_coord.x YGSE[igp] = new_coord.y ZGSE[igp] = new_coord.z Bx[igp] = newB_coord.x By[igp] = newB_coord.y Bz[igp] = newB_coord.z # # now we have the interpolated pressures... get the Kp, L, and MLT # from HOPE Ephemeris # os.chdir('/Users/loisks/Desktop/liemohn10/loisks/EMPHEMERIS_' + sats[iSat]) f2 = glob.glob('rbsp' + lsats[iSat] + '_def_MagEphem_OP77Q_' + date1 + '*.txt')[0] pyf2 = dm.readJSONheadedASCII(f2) # get params LShell = np.array(np.nanmean(np.array(pyf2['L']), axis=1)) # weird multiple rows with L MLT = np.array(pyf2['CDMAG_MLT']) Kp = np.array(KpArr[date1]) # in minutes # # now we sort and compare # 0.5 MLT, 0.25 L, and Kp =1 bins up to kp 9 for iKp in range(9): for iMLT in range(nMLT): for iL in range(nL): temp = np.where((MLT >= MLTbins[iMLT] - 0.25) & (MLT < MLTbins[iMLT] + 0.25))[0] temp2 = np.where((LShell >= Lbins[iL] - .125)
endDt=datetime.datetime.strptime(dateEnd,'%Y%m%d') DT=datetime.datetime.strptime(date, '%Y%m%d') sat=['A', 'B'] aeList=[] alList=[] aeTimeList=[] potentialList=[] dateArr=[] threshold=-10 # # get all the AE data # first get 2013 os.chdir('/Users/loisks/Documents/ResearchProjects/ChargingProject/AEindex/AE_121314/2013') f=glob.glob('*') for iFile in range(len(f)): data=dm.readJSONheadedASCII(f[iFile]) aeList+=list(data['AE']) aeTimeList+=list(data['DateTime']) alList+=list(data['AL']) os.chdir('/Users/loisks/Documents/ResearchProjects/ChargingProject/AEindex/AE_121314/2014') f=glob.glob('*') for iFile in range(len(f)): data=dm.readJSONheadedASCII(f[iFile]) aeList+=list(data['AE']) aeTimeList+=list(data['DateTime']) alList+=list(data['AL']) os.chdir('/Users/loisks/Documents/ResearchProjects/ChargingProject/') # AETimes=np.zeros(len(aeTimeList)) # now have complete lists of times and indexes for iTime in range(len(aeTimeList)):