def _ReadResList():
'''
This function will return a numpy.recarray which lists the
approximate time resolutions of the MAG data between Date0,ut0 and
Date1,ut1.
'''
fname = Globals.ModulePath + '/__data/MagRes/res_list.dat'
tmp = pf.ReadASCIIData(fname,
False,
dtype=[('Date0', 'int32'), ('hhmm', 'int32'),
('Res', 'int32')])
n = tmp.size
dtype = [('Date0', 'int32'), ('ut0', 'float32'), ('Date1', 'int32'),
('ut1', 'int32'), ('Res', 'int32')]
data = np.recarray(n, dtype=dtype)
ut = TT.HHMMteDec(tmp.hhmm, True, True)
data.Date0 = tmp.Date0
data.ut0 = ut
data.Date1[:-1] = tmp.Date0[1:]
data.Date1[-1] = 99999999
data.ut1[:-1] = ut[1:]
data.ut1[-1] = 24.0
data.Res = tmp.res
return data
def ReadData():
'''
Read in the downloaded ASCII file
'''
fname = Globals.MessPath + 'FIPS/ANN/FIPSProtonClass.dat'
return pf.ReadASCIIData(fname, Header=True)
def _ReadCarisma1Hz(fname):
'''
This will read the extracted 1 Hz Carisma data files with the file
name format 'yyyymmddSSSS.F01'.
'''
dtype = [ ('Date','int32'),
('ut','float64'),
('Bx','float64'),
('By','float64'),
('Bz','float64')]
dtype0 = [ ('dt','object'),
('Bx','float64'),
('By','float64'),
('Bz','float64'),
('nothing','object')]
try:
data = pf.ReadASCIIData(fname,Header=True,dtype=dtype0)
n = data.size
except:
lines = pf.ReadASCIIFile(fname)[1:]
n = lines.size
data = np.recarray(n,dtype=dtype0)
data.dt = np.array([l[:14] for l in lines])
for i in range(0,n):
try:
data.Bx[i] = np.float64(l[14:24])
data.By[i] = np.float64(l[24:34])
data.Bz[i] = np.float64(l[34:44])
except:
data.Bx[i] = np.nan
data.By[i] = np.nan
data.Bz[i] = np.nan
out = np.recarray(n,dtype=dtype)
out.Bx = data.Bx
out.By = data.By
out.Bz = data.Bz
out.Date = np.array([np.int32(s[:8]) for s in data.dt])
for i in range(0,n):
h = np.int32(data.dt[i][8:10])
m = np.int32(data.dt[i][10:12])
s = np.int32(data.dt[i][12:14])
out.ut[i] = h + m/60 + s/3600.0
badx = out.Bx > 90000.0
bady = out.By > 90000.0
badz = out.Bz > 90000.0
bad = np.where(badx | bady | badz)[0]
out.Bx[bad] = np.nan
out.By[bad] = np.nan
out.Bz[bad] = np.nan
return out
def _ReadCanopus(fname):
'''
This will read the extracted 0.2 Hz Canopus data files with the file
name format 'yyyymmddSSSS.MAG'.
'''
dtype = [ ('Date','int32'),
('ut','float64'),
('Bx','float64'),
('By','float64'),
('Bz','float64')]
dtype0 = [ ('dt','object'),
('Bx','float64'),
('By','float64'),
('Bz','float64'),
('nothing','object')]
lines = pf.ReadASCIIFile(fname)
n = lines.size
for i in range(0,n):
if lines[i][0] != '#':
break
lines = lines[i+1:]
try:
data = pf.ReadASCIIData(lines.tolist(),Header=False,dtype=dtype0)
except:
return np.recarray(0,dtype=dtype)
n = data.size
out = np.recarray(n,dtype=dtype)
out.Bx = data.Bx
out.By = data.By
out.Bz = data.Bz
out.Date = np.array([np.int32(s[:8]) for s in data.dt])
for i in range(0,n):
h = np.int32(data.dt[i][8:10])
m = np.int32(data.dt[i][10:12])
s = np.int32(data.dt[i][12:14])
out.ut[i] = h + m/60 + s/3600.0
badx = out.Bx > 90000.0
bady = out.By > 90000.0
badz = out.Bz > 90000.0
bad = np.where(badx | bady | badz)[0]
out.Bx[bad] = np.nan
out.By[bad] = np.nan
out.Bz[bad] = np.nan
return out
def ReadSolarRotations():
'''
Read the file containing the solar rotations.
'''
path = Globals.OutputPath + 'Sun/'
fname = path + '/SunRotations.dat'
return pf.ReadASCIIData(fname, dtype=dtype)
def _ReadMSHCrossings():
'''
Reads the file contining the list of crossings of MESSENGER through
the magnetosheath.
Returns:
numpy.recarray
'''
fname = Globals.ModulePath + '__data/MSHCrossings.dat'
dtype = [('Date0', 'int32'), ('ut0', 'float32'), ('Date1', 'int32'),
('ut1', 'float32'), ('dir', 'U1')]
return pf.ReadASCIIData(fname, Header=False, dtype=dtype)
def _ReadMercurySpeed():
'''
Reads the file which stored Mercury's orbital speed from 20080101
to 20150431.
'''
fname = Globals.ModulePath + '/__data/MercurySpeed.dat'
data = pf.ReadASCIIData(fname,
Header=True,
dtype=[('Date', 'int32'), ('utc', 'float64'),
('v', 'float32')])
return data
def _ReadMET():
'''
Reads a data file containing the mission elapsed times (METs) at the
start of every date from 20080101 - 20150430.
Returns:
numpy.recarray
'''
fname = Globals.ModulePath + '__data/MessengerMET.dat'
dtype = [('Date', 'int32'), ('ut', 'float32'), ('MET', 'float64')]
data = pf.ReadASCIIData(fname, dtype=dtype)
return data
def _ReadIAGA2002(fname):
'''
This function will read the IAGA 2002 data format used by
INTERMAGNET.
'''
dtype = [('Date', 'int32'), ('ut', 'float64'), ('Bx', 'float64'),
('By', 'float64'), ('Bz', 'float64')]
dtype0 = [('Date', 'object'), ('ut', 'object'), ('doy', 'int32'),
('Bx', 'float64'), ('By', 'float64'), ('Bz', 'float64'),
('Bm', 'float64')]
lines = pf.ReadASCIIFile(fname)
n = lines.size
for i in range(0, n):
if lines[i][0:4] == 'DATE':
break
lines = lines[i + 1:]
try:
data = pf.ReadASCIIData(lines.tolist(), Header=False, dtype=dtype0)
except:
return np.recarray(0, dtype=dtype)
n = data.size
out = np.recarray(n, dtype=dtype)
out.Bx = data.Bx
out.By = data.By
out.Bz = data.Bz
out.Date = np.array([np.int32(D.replace('-', '')) for D in data.Date])
for i in range(0, n):
h = np.int32(data.ut[i][0:2])
m = np.int32(data.ut[i][3:5])
s = np.float32(data.ut[i][6:])
out.ut[i] = h + m / 60 + s / 3600.0
badx = out.Bx > 90000.0
bady = out.By > 90000.0
badz = out.Bz > 90000.0
bad = np.where(badx | bady | badz)[0]
out.Bx[bad] = np.nan
out.By[bad] = np.nan
out.Bz[bad] = np.nan
return out
def _ReadIndexFile(fname):
'''
This will be reading the index file for a single magnetometer
station.
'''
dtype = [('Date', 'int32'), ('Station', 'object'), ('Res', 'float32'),
('File', 'object'), ('SubDir', 'object')]
if os.path.isfile(fname):
data = pf.ReadASCIIData(fname, Header=True, dtype=dtype)
else:
data = np.recarray(0, dtype=dtype)
return data
def _ReadIMAGE1s(fname):
'''
this is for reading the files with the name 'SSS_yyyymmdd.txt'
'''
dtype0 = [ ('yr','int32'),
('mn','int32'),
('dy','int32'),
('hh','int32'),
('mm','int32'),
('ss','int32'),
('Bx','float64'),
('By','float64'),
('Bz','float64')]
dtype = [ ('Date','int32'),
('ut','float64'),
('Bx','float64'),
('By','float64'),
('Bz','float64')]
try:
data = pf.ReadASCIIData(fname,Header=False,dtype=dtype0)
except:
return np.recarray(0,dtype=dtype)
badx = data.Bx > 90000.0
bady = data.By > 90000.0
badz = data.Bz > 90000.0
bad = np.where(badx | bady | badz)[0]
data.Bx[bad] = np.nan
data.By[bad] = np.nan
data.Bz[bad] = np.nan
n = data.size
out = np.recarray(n,dtype=dtype)
out.Date = TT.DateJoin(data.yr,data.mn,data.dy)
out.ut = TT.HHMMtoDec(data.hh,data.mm,data.ss)
out.Bx = data.Bx
out.By = data.By
out.Bz = data.Bz
return out
def _ReadDataIndex():
'''
Reads an index file containing the file names, update dates and
resolutions.
'''
#define the datatype
dtype = [('FileName', 'object'), ('OldFileName', 'object'),
('UpdateDate', 'object'), ('Res', 'int32')]
#check if the index file exists
fname = Globals.DataPath + 'DataIndex.dat'
if not os.path.isfile(fname):
return np.recarray(0, dtype=dtype)
#if it does exist, then read it
data = pf.ReadASCIIData(fname, Header=True, dtype=dtype)
return data
def ReadPDSMAG(fname):
'''
This will read in the .TAB datafile and output a numpy.recarray
Inputs:
fname: file name and path
'''
pdsdtype = [ ('Year','int32'),
('DOY','int32'),
('Hour','int32'),
('Min','int32'),
('Sec','float32'),
('MET','float32'),
('Xmso','float32'),
('Ymso','float32'),
('Zmso','float32'),
('Bx','float32'),
('By','float32'),
('Bz','float32'),
('Loc','U2')]
data = pf.ReadASCIIData(fname,False,dtype=pdsdtype)
dtype = MagGlobals.dtypes['MSM']
n = data.size
out = np.recarray(n,dtype)
out.Date = np.array([TT.DayNotoDate(data.Year[i],data.DOY[i])[0] for i in range(0,n)])
out.ut = np.float32(data.Hour) + np.float32(data.Min)/60.0 + np.float32(data.Sec)/3600.0
out.utc = ContUT(out.Date,out.ut)
out.Xmso = data.Xmso/2440.0
out.Ymso = data.Ymso/2440.0
out.Zmso = data.Zmso/2440.0
out.Xmsm = data.Xmso/2440.0
out.Ymsm = data.Ymso/2440.0
out.Zmsm = data.Zmso/2440.0 - 0.196
out.Bx = data.Bx
out.By = data.By
out.Bz = data.Bz
out.Loc = GetRegion(out.Date,out.ut,out.utc,Verbose=False)
return out
def _ReadDataIndex(sc='a'):
'''
Reads index file containing a list of all of the dates with their
associated data file name (so that we can pick the version
automatically).
'''
#define the dtype
dtype = [('Date', 'int32'), ('FileName', 'object'), ('Version', 'int16')]
#find the file
fname = Globals.DataPath + 'MagEph/{:s}.dat'.format(sc)
#check it exists
if not os.path.isfile(fname):
return np.recarray(0, dtype=dtype)
#read the index file
data = pf.ReadASCIIData(fname, True, dtype=dtype)
return data
def _ReadDataIndex(fname):
'''
Reads index file containing a list of all of the dates with their
associated data file name (so that we can pick the version
automatically).
'''
#define the dtype
dtype = [('Date', 'int32'), ('FileName', 'object'), ('Version', 'int16')]
#check it exists
if not os.path.isfile(fname):
return np.recarray(0, dtype=dtype)
#read the index file
try:
data = pf.ReadASCIIData(fname, True, dtype=dtype)
except:
return np.recarray(0, dtype=dtype)
return data
def _ReadMSTimes():
#the file name
fname = Globals.ModuleData + 'MSTimes.dat'
return pf.ReadASCIIData(fname)
import numpy as np
import PyFileIO as pf
from .. import Globals
import DateTimeTools as TT
def _ReadResList()
'''
This function will return a numpy.recarray which lists the
approximate time resolutions of the MAG data between Date0,ut0 and
Date1,ut1.
'''
fname = Globals.ModulePath+'/__data/MagRes/res_list.dat'
tmp = pf.ReadASCIIData(fname,False,dtype=[('Date0','int32'),('hhmm','int32'),('Res','int32')])
n = tmp.size
dtype = [('Date0','int32'),('ut0','float32'),('Date1','int32'),('ut1','int32'),('Res','int32')]
data = np.recarray(n,dtype=dtype)
ut = TT.HHMMteDec(tmp.hhmm,True,True)
data.Date0 = tmp.Date0
data.ut0 = ut
data.Date1[:-1] = tmp.Date0[1:]
data.Date1[-1] = 99999999
data.ut1[:-1] = ut[1:]
data.ut1[-1] = 24.0
data.Res = tmp.res
return data
def ReadCarringtonRotations(): path = Globals.OutputPath + 'Venus/' fname = path + '0long.dat' return pf.ReadASCIIData(fname,Header=False,dtype=dtypecarr)
def _ReadMPCrossings():
fname = Globals.ModulePath + '__data/MessMP.dat'
data = pf.ReadASCIIData(fname)
return data
def _ConvertFTPFile(FullPath,fname,UpdateDate,Res):
'''
Converts standard OMNI ASCII files to binary, replaces missing data
flags with NAN where possible.
Inputs:
FullPath: full apth and file name of the input file
fname: just the name of the input file
UpdateDate: The date which this was last upated on the OMNI site.
Res: time resolution (1 or 5)
'''
#define the data type
ErrVal = {'Year':999,
'DOY':999,
'Hour':99,
'Minute':99,
'SC_IMF':99,
'SC_Plasma':99,
'N_IMF':999,
'N_Plasma':999,
'PercInterp':999,
'TimeShift':999999,
'RMSTimeShift':999999,
'RMSPhaseFrontNorm':99.99,
'dTime':999999,
'B':9999.99,
'BxGSE':9999.99,
'ByGSE':9999.99,
'BzGSE':9999.99,
'ByGSM':9999.99,
'BzGSM':9999.99,
'RMSSDBScalar':9999.99,
'RMSSDFieldVector':9999.99,
'FlowSpeed':99999.9,
'Vx':99999.9,
'Vy':99999.9,
'Vz':99999.9,
'ProtonDensity':999.99,
'Temp':9999999.0,
'FlowPressure':99.99,
'E':999.99,
'Beta':999.99,
'MA':999.9,
'Xsc':9999.99,
'Ysc':9999.99,
'Zsc':9999.99,
'Xbsn':9999.99,
'Ybsn':9999.99,
'Zbsn':9999.99,
'AE':99999,
'AL':99999,
'AU':99999,
'SymD':99999,
'SymH':99999,
'AsyD':99999,
'AsyH':99999,
'PC':999.99,
'Ms':99.9,
'Pflux10':99999.99,
'Pflux30':99999.99,
'Pflux60':99999.99}
if Res == 1:
indtype = [('Year','int32'),('DOY','int32'),('Hour','int32'),('Minute','int32'),
('SC_IMF','uint8'),('SC_Plasma','uint8'),('N_IMF','int32'),('N_Plasma','int32'),
('PercInterp','float32'),('TimeShift','float32'),('RMSTimeShift','float32'),
('RMSPhaseFrontNorm','float32'), ('dTime','float32'),('B','float32'),
('BxGSE','float32'),('ByGSE','float32'),('BzGSE','float32'),('ByGSM','float32'),
('BzGSM','float32'),('RMSSDBScalar','float32'),('RMSSDFieldVector','float32'),
('FlowSpeed','float32'),('Vx','float32'),('Vy','float32'),('Vz','float32'),
('ProtonDensity','float32'),('Temp','float32'),('FlowPressure','float32'),
('E','float32'),('Beta','float32'),('MA','float32'),('Xsc','float32'),
('Ysc','float32'),('Zsc','float32'),('Xbsn','float32'),('Ybsn','float32'),
('Zbsn','float32'),('AE','float32'),('AL','float32'),('AU','float32'),
('SymD','float32'),('SymH','float32'),('AsyD','float32'),('AsyH','float32'),
('PC','float32'),('Ms','float32')]
outdtype = Globals.dtype1
else:
indtype = [('Year','int32'),('DOY','int32'),('Hour','int32'),('Minute','int32'),
('SC_IMF','uint8'),('SC_Plasma','uint8'),('N_IMF','int32'),('N_Plasma','int32'),
('PercInterp','float32'),('TimeShift','float32'),('RMSTimeShift','float32'),
('RMSPhaseFrontNorm','float32'), ('dTime','float32'),('B','float32'),
('BxGSE','float32'),('ByGSE','float32'),('BzGSE','float32'),('ByGSM','float32'),
('BzGSM','float32'),('RMSSDBScalar','float32'),('RMSSDFieldVector','float32'),
('FlowSpeed','float32'),('Vx','float32'),('Vy','float32'),('Vz','float32'),
('ProtonDensity','float32'),('Temp','float32'),('FlowPressure','float32'),
('E','float32'),('Beta','float32'),('MA','float32'),('Xsc','float32'),
('Ysc','float32'),('Zsc','float32'),('Xbsn','float32'),('Ybsn','float32'),
('Zbsn','float32'),('AE','float32'),('AL','float32'),('AU','float32'),
('SymD','float32'),('SymH','float32'),('AsyD','float32'),('AsyH','float32'),
('PC','float32'),('Ms','float32'),('Pflux10','float32'),('Pflux30','float32'),
('Pflux60','float32')]
outdtype = Globals.dtype5
#read the file
data = pf.ReadASCIIData(FullPath,Header=False,dtype=indtype)
#create output array
n = np.size(data)
out = np.recarray(n,dtype=outdtype)
nf = np.size(out.dtype.names)
progress = 0
print('\r[' + '='*progress + '-'*(nf-progress)+']',end='')
#convert dates
out.Date = TT.DayNotoDate(data.Year,data.DOY)
progress += 1
print('\r[' + '='*progress + '-'*(nf-progress)+']',end='')
#convert time
out.ut = data.Hour + data.Minute/60.0
progress += 1
print('\r[' + '='*progress + '-'*(nf-progress)+']',end='')
out.utc = TT.ContUT(out.Date,out.ut)
progress += 1
print('\r[' + '='*progress + '-'*(nf-progress)+']',end='')
#copy new data across
fields = data.dtype.names
for f in fields:
if f in out.dtype.names:
out[f] = data[f]
bad = np.where(out[f] == ErrVal[f])[0]
if bad.size > 0:
if 'float' in str(out[f].dtype):
out[f][bad] = np.nan
progress += 1
print('\r[' + '='*progress + '-'*(nf-progress)+']',end='')
print()
#get the year from the file name
Year = np.int32(fname[-8:-4])
#save file
outfname = 'OMNI-{:1d}min-{:4d}.bin'.format(Res,Year)
outpath = Globals.DataPath+'{:1d}/'.format(Res)
if not os.path.isdir(outpath):
os.makedirs(outpath)
print('Saving')
RT.SaveRecarray(out,outpath+outfname,Progress=True)
print('Saved file: '+outfname)
#update index
idx = _ReadDataIndex()
use = np.where(idx.OldFileName == fname)[0]
if use.size == 0:
#this file does not yet exist within the index file
tmp = np.recarray(1,dtype=idx.dtype)
tmp[0].FileName = outfname
tmp[0].OldFileName = fname
tmp[0].UpdateDate = UpdateDate
tmp[0].Res = Res
idx = RT.JoinRecarray(idx,tmp)
else:
#file exists, update the information
idx[use[0]].FileName = outfname
idx[use[0]].OldFileName = fname
idx[use[0]].UpdateDate = UpdateDate
idx[use[0]].Res = Res
_UpdateDataIndex(idx)