def _ReadResList(): ''' This function will return a numpy.recarray which lists the approximate time resolutions of the MAG data between Date0,ut0 and Date1,ut1. ''' fname = Globals.ModulePath + '/__data/MagRes/res_list.dat' tmp = pf.ReadASCIIData(fname, False, dtype=[('Date0', 'int32'), ('hhmm', 'int32'), ('Res', 'int32')]) n = tmp.size dtype = [('Date0', 'int32'), ('ut0', 'float32'), ('Date1', 'int32'), ('ut1', 'int32'), ('Res', 'int32')] data = np.recarray(n, dtype=dtype) ut = TT.HHMMteDec(tmp.hhmm, True, True) data.Date0 = tmp.Date0 data.ut0 = ut data.Date1[:-1] = tmp.Date0[1:] data.Date1[-1] = 99999999 data.ut1[:-1] = ut[1:] data.ut1[-1] = 24.0 data.Res = tmp.res return data
def ReadData(): ''' Read in the downloaded ASCII file ''' fname = Globals.MessPath + 'FIPS/ANN/FIPSProtonClass.dat' return pf.ReadASCIIData(fname, Header=True)
def _ReadCarisma1Hz(fname): ''' This will read the extracted 1 Hz Carisma data files with the file name format 'yyyymmddSSSS.F01'. ''' dtype = [ ('Date','int32'), ('ut','float64'), ('Bx','float64'), ('By','float64'), ('Bz','float64')] dtype0 = [ ('dt','object'), ('Bx','float64'), ('By','float64'), ('Bz','float64'), ('nothing','object')] try: data = pf.ReadASCIIData(fname,Header=True,dtype=dtype0) n = data.size except: lines = pf.ReadASCIIFile(fname)[1:] n = lines.size data = np.recarray(n,dtype=dtype0) data.dt = np.array([l[:14] for l in lines]) for i in range(0,n): try: data.Bx[i] = np.float64(l[14:24]) data.By[i] = np.float64(l[24:34]) data.Bz[i] = np.float64(l[34:44]) except: data.Bx[i] = np.nan data.By[i] = np.nan data.Bz[i] = np.nan out = np.recarray(n,dtype=dtype) out.Bx = data.Bx out.By = data.By out.Bz = data.Bz out.Date = np.array([np.int32(s[:8]) for s in data.dt]) for i in range(0,n): h = np.int32(data.dt[i][8:10]) m = np.int32(data.dt[i][10:12]) s = np.int32(data.dt[i][12:14]) out.ut[i] = h + m/60 + s/3600.0 badx = out.Bx > 90000.0 bady = out.By > 90000.0 badz = out.Bz > 90000.0 bad = np.where(badx | bady | badz)[0] out.Bx[bad] = np.nan out.By[bad] = np.nan out.Bz[bad] = np.nan return out
def _ReadCanopus(fname): ''' This will read the extracted 0.2 Hz Canopus data files with the file name format 'yyyymmddSSSS.MAG'. ''' dtype = [ ('Date','int32'), ('ut','float64'), ('Bx','float64'), ('By','float64'), ('Bz','float64')] dtype0 = [ ('dt','object'), ('Bx','float64'), ('By','float64'), ('Bz','float64'), ('nothing','object')] lines = pf.ReadASCIIFile(fname) n = lines.size for i in range(0,n): if lines[i][0] != '#': break lines = lines[i+1:] try: data = pf.ReadASCIIData(lines.tolist(),Header=False,dtype=dtype0) except: return np.recarray(0,dtype=dtype) n = data.size out = np.recarray(n,dtype=dtype) out.Bx = data.Bx out.By = data.By out.Bz = data.Bz out.Date = np.array([np.int32(s[:8]) for s in data.dt]) for i in range(0,n): h = np.int32(data.dt[i][8:10]) m = np.int32(data.dt[i][10:12]) s = np.int32(data.dt[i][12:14]) out.ut[i] = h + m/60 + s/3600.0 badx = out.Bx > 90000.0 bady = out.By > 90000.0 badz = out.Bz > 90000.0 bad = np.where(badx | bady | badz)[0] out.Bx[bad] = np.nan out.By[bad] = np.nan out.Bz[bad] = np.nan return out
def ReadSolarRotations(): ''' Read the file containing the solar rotations. ''' path = Globals.OutputPath + 'Sun/' fname = path + '/SunRotations.dat' return pf.ReadASCIIData(fname, dtype=dtype)
def _ReadMSHCrossings(): ''' Reads the file contining the list of crossings of MESSENGER through the magnetosheath. Returns: numpy.recarray ''' fname = Globals.ModulePath + '__data/MSHCrossings.dat' dtype = [('Date0', 'int32'), ('ut0', 'float32'), ('Date1', 'int32'), ('ut1', 'float32'), ('dir', 'U1')] return pf.ReadASCIIData(fname, Header=False, dtype=dtype)
def _ReadMercurySpeed(): ''' Reads the file which stored Mercury's orbital speed from 20080101 to 20150431. ''' fname = Globals.ModulePath + '/__data/MercurySpeed.dat' data = pf.ReadASCIIData(fname, Header=True, dtype=[('Date', 'int32'), ('utc', 'float64'), ('v', 'float32')]) return data
def _ReadMET(): ''' Reads a data file containing the mission elapsed times (METs) at the start of every date from 20080101 - 20150430. Returns: numpy.recarray ''' fname = Globals.ModulePath + '__data/MessengerMET.dat' dtype = [('Date', 'int32'), ('ut', 'float32'), ('MET', 'float64')] data = pf.ReadASCIIData(fname, dtype=dtype) return data
def _ReadIAGA2002(fname): ''' This function will read the IAGA 2002 data format used by INTERMAGNET. ''' dtype = [('Date', 'int32'), ('ut', 'float64'), ('Bx', 'float64'), ('By', 'float64'), ('Bz', 'float64')] dtype0 = [('Date', 'object'), ('ut', 'object'), ('doy', 'int32'), ('Bx', 'float64'), ('By', 'float64'), ('Bz', 'float64'), ('Bm', 'float64')] lines = pf.ReadASCIIFile(fname) n = lines.size for i in range(0, n): if lines[i][0:4] == 'DATE': break lines = lines[i + 1:] try: data = pf.ReadASCIIData(lines.tolist(), Header=False, dtype=dtype0) except: return np.recarray(0, dtype=dtype) n = data.size out = np.recarray(n, dtype=dtype) out.Bx = data.Bx out.By = data.By out.Bz = data.Bz out.Date = np.array([np.int32(D.replace('-', '')) for D in data.Date]) for i in range(0, n): h = np.int32(data.ut[i][0:2]) m = np.int32(data.ut[i][3:5]) s = np.float32(data.ut[i][6:]) out.ut[i] = h + m / 60 + s / 3600.0 badx = out.Bx > 90000.0 bady = out.By > 90000.0 badz = out.Bz > 90000.0 bad = np.where(badx | bady | badz)[0] out.Bx[bad] = np.nan out.By[bad] = np.nan out.Bz[bad] = np.nan return out
def _ReadIndexFile(fname): ''' This will be reading the index file for a single magnetometer station. ''' dtype = [('Date', 'int32'), ('Station', 'object'), ('Res', 'float32'), ('File', 'object'), ('SubDir', 'object')] if os.path.isfile(fname): data = pf.ReadASCIIData(fname, Header=True, dtype=dtype) else: data = np.recarray(0, dtype=dtype) return data
def _ReadIMAGE1s(fname): ''' this is for reading the files with the name 'SSS_yyyymmdd.txt' ''' dtype0 = [ ('yr','int32'), ('mn','int32'), ('dy','int32'), ('hh','int32'), ('mm','int32'), ('ss','int32'), ('Bx','float64'), ('By','float64'), ('Bz','float64')] dtype = [ ('Date','int32'), ('ut','float64'), ('Bx','float64'), ('By','float64'), ('Bz','float64')] try: data = pf.ReadASCIIData(fname,Header=False,dtype=dtype0) except: return np.recarray(0,dtype=dtype) badx = data.Bx > 90000.0 bady = data.By > 90000.0 badz = data.Bz > 90000.0 bad = np.where(badx | bady | badz)[0] data.Bx[bad] = np.nan data.By[bad] = np.nan data.Bz[bad] = np.nan n = data.size out = np.recarray(n,dtype=dtype) out.Date = TT.DateJoin(data.yr,data.mn,data.dy) out.ut = TT.HHMMtoDec(data.hh,data.mm,data.ss) out.Bx = data.Bx out.By = data.By out.Bz = data.Bz return out
def _ReadDataIndex(): ''' Reads an index file containing the file names, update dates and resolutions. ''' #define the datatype dtype = [('FileName', 'object'), ('OldFileName', 'object'), ('UpdateDate', 'object'), ('Res', 'int32')] #check if the index file exists fname = Globals.DataPath + 'DataIndex.dat' if not os.path.isfile(fname): return np.recarray(0, dtype=dtype) #if it does exist, then read it data = pf.ReadASCIIData(fname, Header=True, dtype=dtype) return data
def ReadPDSMAG(fname): ''' This will read in the .TAB datafile and output a numpy.recarray Inputs: fname: file name and path ''' pdsdtype = [ ('Year','int32'), ('DOY','int32'), ('Hour','int32'), ('Min','int32'), ('Sec','float32'), ('MET','float32'), ('Xmso','float32'), ('Ymso','float32'), ('Zmso','float32'), ('Bx','float32'), ('By','float32'), ('Bz','float32'), ('Loc','U2')] data = pf.ReadASCIIData(fname,False,dtype=pdsdtype) dtype = MagGlobals.dtypes['MSM'] n = data.size out = np.recarray(n,dtype) out.Date = np.array([TT.DayNotoDate(data.Year[i],data.DOY[i])[0] for i in range(0,n)]) out.ut = np.float32(data.Hour) + np.float32(data.Min)/60.0 + np.float32(data.Sec)/3600.0 out.utc = ContUT(out.Date,out.ut) out.Xmso = data.Xmso/2440.0 out.Ymso = data.Ymso/2440.0 out.Zmso = data.Zmso/2440.0 out.Xmsm = data.Xmso/2440.0 out.Ymsm = data.Ymso/2440.0 out.Zmsm = data.Zmso/2440.0 - 0.196 out.Bx = data.Bx out.By = data.By out.Bz = data.Bz out.Loc = GetRegion(out.Date,out.ut,out.utc,Verbose=False) return out
def _ReadDataIndex(sc='a'): ''' Reads index file containing a list of all of the dates with their associated data file name (so that we can pick the version automatically). ''' #define the dtype dtype = [('Date', 'int32'), ('FileName', 'object'), ('Version', 'int16')] #find the file fname = Globals.DataPath + 'MagEph/{:s}.dat'.format(sc) #check it exists if not os.path.isfile(fname): return np.recarray(0, dtype=dtype) #read the index file data = pf.ReadASCIIData(fname, True, dtype=dtype) return data
def _ReadDataIndex(fname): ''' Reads index file containing a list of all of the dates with their associated data file name (so that we can pick the version automatically). ''' #define the dtype dtype = [('Date', 'int32'), ('FileName', 'object'), ('Version', 'int16')] #check it exists if not os.path.isfile(fname): return np.recarray(0, dtype=dtype) #read the index file try: data = pf.ReadASCIIData(fname, True, dtype=dtype) except: return np.recarray(0, dtype=dtype) return data
def _ReadMSTimes(): #the file name fname = Globals.ModuleData + 'MSTimes.dat' return pf.ReadASCIIData(fname)
import numpy as np import PyFileIO as pf from .. import Globals import DateTimeTools as TT def _ReadResList() ''' This function will return a numpy.recarray which lists the approximate time resolutions of the MAG data between Date0,ut0 and Date1,ut1. ''' fname = Globals.ModulePath+'/__data/MagRes/res_list.dat' tmp = pf.ReadASCIIData(fname,False,dtype=[('Date0','int32'),('hhmm','int32'),('Res','int32')]) n = tmp.size dtype = [('Date0','int32'),('ut0','float32'),('Date1','int32'),('ut1','int32'),('Res','int32')] data = np.recarray(n,dtype=dtype) ut = TT.HHMMteDec(tmp.hhmm,True,True) data.Date0 = tmp.Date0 data.ut0 = ut data.Date1[:-1] = tmp.Date0[1:] data.Date1[-1] = 99999999 data.ut1[:-1] = ut[1:] data.ut1[-1] = 24.0 data.Res = tmp.res return data
def ReadCarringtonRotations(): path = Globals.OutputPath + 'Venus/' fname = path + '0long.dat' return pf.ReadASCIIData(fname,Header=False,dtype=dtypecarr)
def _ReadMPCrossings(): fname = Globals.ModulePath + '__data/MessMP.dat' data = pf.ReadASCIIData(fname) return data
def _ConvertFTPFile(FullPath,fname,UpdateDate,Res): ''' Converts standard OMNI ASCII files to binary, replaces missing data flags with NAN where possible. Inputs: FullPath: full apth and file name of the input file fname: just the name of the input file UpdateDate: The date which this was last upated on the OMNI site. Res: time resolution (1 or 5) ''' #define the data type ErrVal = {'Year':999, 'DOY':999, 'Hour':99, 'Minute':99, 'SC_IMF':99, 'SC_Plasma':99, 'N_IMF':999, 'N_Plasma':999, 'PercInterp':999, 'TimeShift':999999, 'RMSTimeShift':999999, 'RMSPhaseFrontNorm':99.99, 'dTime':999999, 'B':9999.99, 'BxGSE':9999.99, 'ByGSE':9999.99, 'BzGSE':9999.99, 'ByGSM':9999.99, 'BzGSM':9999.99, 'RMSSDBScalar':9999.99, 'RMSSDFieldVector':9999.99, 'FlowSpeed':99999.9, 'Vx':99999.9, 'Vy':99999.9, 'Vz':99999.9, 'ProtonDensity':999.99, 'Temp':9999999.0, 'FlowPressure':99.99, 'E':999.99, 'Beta':999.99, 'MA':999.9, 'Xsc':9999.99, 'Ysc':9999.99, 'Zsc':9999.99, 'Xbsn':9999.99, 'Ybsn':9999.99, 'Zbsn':9999.99, 'AE':99999, 'AL':99999, 'AU':99999, 'SymD':99999, 'SymH':99999, 'AsyD':99999, 'AsyH':99999, 'PC':999.99, 'Ms':99.9, 'Pflux10':99999.99, 'Pflux30':99999.99, 'Pflux60':99999.99} if Res == 1: indtype = [('Year','int32'),('DOY','int32'),('Hour','int32'),('Minute','int32'), ('SC_IMF','uint8'),('SC_Plasma','uint8'),('N_IMF','int32'),('N_Plasma','int32'), ('PercInterp','float32'),('TimeShift','float32'),('RMSTimeShift','float32'), ('RMSPhaseFrontNorm','float32'), ('dTime','float32'),('B','float32'), ('BxGSE','float32'),('ByGSE','float32'),('BzGSE','float32'),('ByGSM','float32'), ('BzGSM','float32'),('RMSSDBScalar','float32'),('RMSSDFieldVector','float32'), ('FlowSpeed','float32'),('Vx','float32'),('Vy','float32'),('Vz','float32'), ('ProtonDensity','float32'),('Temp','float32'),('FlowPressure','float32'), ('E','float32'),('Beta','float32'),('MA','float32'),('Xsc','float32'), ('Ysc','float32'),('Zsc','float32'),('Xbsn','float32'),('Ybsn','float32'), ('Zbsn','float32'),('AE','float32'),('AL','float32'),('AU','float32'), ('SymD','float32'),('SymH','float32'),('AsyD','float32'),('AsyH','float32'), ('PC','float32'),('Ms','float32')] outdtype = Globals.dtype1 else: indtype = [('Year','int32'),('DOY','int32'),('Hour','int32'),('Minute','int32'), ('SC_IMF','uint8'),('SC_Plasma','uint8'),('N_IMF','int32'),('N_Plasma','int32'), ('PercInterp','float32'),('TimeShift','float32'),('RMSTimeShift','float32'), ('RMSPhaseFrontNorm','float32'), ('dTime','float32'),('B','float32'), ('BxGSE','float32'),('ByGSE','float32'),('BzGSE','float32'),('ByGSM','float32'), ('BzGSM','float32'),('RMSSDBScalar','float32'),('RMSSDFieldVector','float32'), ('FlowSpeed','float32'),('Vx','float32'),('Vy','float32'),('Vz','float32'), ('ProtonDensity','float32'),('Temp','float32'),('FlowPressure','float32'), ('E','float32'),('Beta','float32'),('MA','float32'),('Xsc','float32'), ('Ysc','float32'),('Zsc','float32'),('Xbsn','float32'),('Ybsn','float32'), ('Zbsn','float32'),('AE','float32'),('AL','float32'),('AU','float32'), ('SymD','float32'),('SymH','float32'),('AsyD','float32'),('AsyH','float32'), ('PC','float32'),('Ms','float32'),('Pflux10','float32'),('Pflux30','float32'), ('Pflux60','float32')] outdtype = Globals.dtype5 #read the file data = pf.ReadASCIIData(FullPath,Header=False,dtype=indtype) #create output array n = np.size(data) out = np.recarray(n,dtype=outdtype) nf = np.size(out.dtype.names) progress = 0 print('\r[' + '='*progress + '-'*(nf-progress)+']',end='') #convert dates out.Date = TT.DayNotoDate(data.Year,data.DOY) progress += 1 print('\r[' + '='*progress + '-'*(nf-progress)+']',end='') #convert time out.ut = data.Hour + data.Minute/60.0 progress += 1 print('\r[' + '='*progress + '-'*(nf-progress)+']',end='') out.utc = TT.ContUT(out.Date,out.ut) progress += 1 print('\r[' + '='*progress + '-'*(nf-progress)+']',end='') #copy new data across fields = data.dtype.names for f in fields: if f in out.dtype.names: out[f] = data[f] bad = np.where(out[f] == ErrVal[f])[0] if bad.size > 0: if 'float' in str(out[f].dtype): out[f][bad] = np.nan progress += 1 print('\r[' + '='*progress + '-'*(nf-progress)+']',end='') print() #get the year from the file name Year = np.int32(fname[-8:-4]) #save file outfname = 'OMNI-{:1d}min-{:4d}.bin'.format(Res,Year) outpath = Globals.DataPath+'{:1d}/'.format(Res) if not os.path.isdir(outpath): os.makedirs(outpath) print('Saving') RT.SaveRecarray(out,outpath+outfname,Progress=True) print('Saved file: '+outfname) #update index idx = _ReadDataIndex() use = np.where(idx.OldFileName == fname)[0] if use.size == 0: #this file does not yet exist within the index file tmp = np.recarray(1,dtype=idx.dtype) tmp[0].FileName = outfname tmp[0].OldFileName = fname tmp[0].UpdateDate = UpdateDate tmp[0].Res = Res idx = RT.JoinRecarray(idx,tmp) else: #file exists, update the information idx[use[0]].FileName = outfname idx[use[0]].OldFileName = fname idx[use[0]].UpdateDate = UpdateDate idx[use[0]].Res = Res _UpdateDataIndex(idx)