def _ReadCarisma1Hz(fname):
'''
This will read the extracted 1 Hz Carisma data files with the file
name format 'yyyymmddSSSS.F01'.
'''
dtype = [ ('Date','int32'),
('ut','float64'),
('Bx','float64'),
('By','float64'),
('Bz','float64')]
dtype0 = [ ('dt','object'),
('Bx','float64'),
('By','float64'),
('Bz','float64'),
('nothing','object')]
try:
data = pf.ReadASCIIData(fname,Header=True,dtype=dtype0)
n = data.size
except:
lines = pf.ReadASCIIFile(fname)[1:]
n = lines.size
data = np.recarray(n,dtype=dtype0)
data.dt = np.array([l[:14] for l in lines])
for i in range(0,n):
try:
data.Bx[i] = np.float64(l[14:24])
data.By[i] = np.float64(l[24:34])
data.Bz[i] = np.float64(l[34:44])
except:
data.Bx[i] = np.nan
data.By[i] = np.nan
data.Bz[i] = np.nan
out = np.recarray(n,dtype=dtype)
out.Bx = data.Bx
out.By = data.By
out.Bz = data.Bz
out.Date = np.array([np.int32(s[:8]) for s in data.dt])
for i in range(0,n):
h = np.int32(data.dt[i][8:10])
m = np.int32(data.dt[i][10:12])
s = np.int32(data.dt[i][12:14])
out.ut[i] = h + m/60 + s/3600.0
badx = out.Bx > 90000.0
bady = out.By > 90000.0
badz = out.Bz > 90000.0
bad = np.where(badx | bady | badz)[0]
out.Bx[bad] = np.nan
out.By[bad] = np.nan
out.Bz[bad] = np.nan
return out
def _ReadCanopus(fname):
'''
This will read the extracted 0.2 Hz Canopus data files with the file
name format 'yyyymmddSSSS.MAG'.
'''
dtype = [ ('Date','int32'),
('ut','float64'),
('Bx','float64'),
('By','float64'),
('Bz','float64')]
dtype0 = [ ('dt','object'),
('Bx','float64'),
('By','float64'),
('Bz','float64'),
('nothing','object')]
lines = pf.ReadASCIIFile(fname)
n = lines.size
for i in range(0,n):
if lines[i][0] != '#':
break
lines = lines[i+1:]
try:
data = pf.ReadASCIIData(lines.tolist(),Header=False,dtype=dtype0)
except:
return np.recarray(0,dtype=dtype)
n = data.size
out = np.recarray(n,dtype=dtype)
out.Bx = data.Bx
out.By = data.By
out.Bz = data.Bz
out.Date = np.array([np.int32(s[:8]) for s in data.dt])
for i in range(0,n):
h = np.int32(data.dt[i][8:10])
m = np.int32(data.dt[i][10:12])
s = np.int32(data.dt[i][12:14])
out.ut[i] = h + m/60 + s/3600.0
badx = out.Bx > 90000.0
bady = out.By > 90000.0
badz = out.Bz > 90000.0
bad = np.where(badx | bady | badz)[0]
out.Bx[bad] = np.nan
out.By[bad] = np.nan
out.Bz[bad] = np.nan
return out
def _ReadIAGA2002(fname):
'''
This function will read the IAGA 2002 data format used by
INTERMAGNET.
'''
dtype = [('Date', 'int32'), ('ut', 'float64'), ('Bx', 'float64'),
('By', 'float64'), ('Bz', 'float64')]
dtype0 = [('Date', 'object'), ('ut', 'object'), ('doy', 'int32'),
('Bx', 'float64'), ('By', 'float64'), ('Bz', 'float64'),
('Bm', 'float64')]
lines = pf.ReadASCIIFile(fname)
n = lines.size
for i in range(0, n):
if lines[i][0:4] == 'DATE':
break
lines = lines[i + 1:]
try:
data = pf.ReadASCIIData(lines.tolist(), Header=False, dtype=dtype0)
except:
return np.recarray(0, dtype=dtype)
n = data.size
out = np.recarray(n, dtype=dtype)
out.Bx = data.Bx
out.By = data.By
out.Bz = data.Bz
out.Date = np.array([np.int32(D.replace('-', '')) for D in data.Date])
for i in range(0, n):
h = np.int32(data.ut[i][0:2])
m = np.int32(data.ut[i][3:5])
s = np.float32(data.ut[i][6:])
out.ut[i] = h + m / 60 + s / 3600.0
badx = out.Bx > 90000.0
bady = out.By > 90000.0
badz = out.Bz > 90000.0
bad = np.where(badx | bady | badz)[0]
out.Bx[bad] = np.nan
out.By[bad] = np.nan
out.Bz[bad] = np.nan
return out
def ReadMirrorAlt(Date, path):
'''
Read a PAD file
'''
#get the file name
fname = path + '{:08d}/'.format(Date) + 'Mirror.bin'
#check it exists
if not os.path.isfile(fname):
print('File not found')
return None
#read the data
f = open(fname, 'rb')
out = {}
keys = [
'Date', 'ut', 'utc', 'Alt', 'AltMid', 'Bm', 'BmMid', 'B0', 'AlphaN',
'AlphaS', 'BaltN', 'BaltS', 'LCAlt'
]
for k in keys:
if k == 'utc':
dtype = 'float64'
else:
dtype = 'float32'
out[k] = pf.ArrayFromFile(dtype, f)
f.close()
return out
def _ReadResList():
'''
This function will return a numpy.recarray which lists the
approximate time resolutions of the MAG data between Date0,ut0 and
Date1,ut1.
'''
fname = Globals.ModulePath + '/__data/MagRes/res_list.dat'
tmp = pf.ReadASCIIData(fname,
False,
dtype=[('Date0', 'int32'), ('hhmm', 'int32'),
('Res', 'int32')])
n = tmp.size
dtype = [('Date0', 'int32'), ('ut0', 'float32'), ('Date1', 'int32'),
('ut1', 'int32'), ('Res', 'int32')]
data = np.recarray(n, dtype=dtype)
ut = TT.HHMMteDec(tmp.hhmm, True, True)
data.Date0 = tmp.Date0
data.ut0 = ut
data.Date1[:-1] = tmp.Date0[1:]
data.Date1[-1] = 99999999
data.ut1[:-1] = ut[1:]
data.ut1[-1] = 24.0
data.Res = tmp.res
return data
def _ReadIMAGEiaga(fname):
'''
Read the mess of a format that is iaga.
Name format: 'image_yyyymmdd.iaga' (10s data)
'''
#read the data
lines = pf.ReadASCIIFile(fname)
s = lines[0]
#output dict
out = {}
#split into records
rlen = np.int32(s[:4])
ss = np.array([s[i*rlen:(i+1)*rlen] for i in range(0,len(s)//rlen)])
nr = len(ss)
#get ID
ID = np.array([ss[i][12:15] for i in range(0,nr)])
IDu = np.unique(ID)
for I in IDu:
use = np.where(ID == I)[0]
out[I] = _iaga_station(ss[use])
return out
def _ReadStations(Network):
'''
Read the list of stations for a magnetometer network.
'''
fname = Globals.StationPath + Network + '.list'
dtype = [('Code', 'U4'), ('Station', 'object'), ('Network', 'object'),
('glat', 'float32'), ('glon', 'float32'), ('mlat', 'float32'),
('mlon', 'float32'), ('L', 'float32')]
data = pf.ReadASCIIFile(fname)
n = data.size
out = np.recarray(n, dtype=dtype)
for i in range(0, n):
s = data[i].split()
#the number of bits which are the name
ln = len(s) - 6
out[i].Code = s[0]
out[i].Station = ' '.join(s[1:ln + 1])
out[i].glat = np.float32(s[1 + ln])
out[i].glon = np.float32(s[2 + ln])
out[i].mlat = np.float32(s[3 + ln])
out[i].mlon = np.float32(s[4 + ln])
out[i].L = np.float32(s[5 + ln])
out.Network[:] = Network
return out
def SaveMirrorAlt(Date, path, Mirror, Overwrite=False):
'''
Save mirror altitudes and fields to go with the pitch angle
distribution data
'''
#create the output path
outpath = path + '{:08d}/'.format(Date)
if not os.path.isdir(outpath):
os.system('mkdir -pv ' + outpath)
os.system('chmod 777 ' + outpath)
#loop through and save each one
fname = outpath + 'Mirror.bin'
if os.path.isfile(fname) and not Overwrite:
return
print('saving file: {:s}'.format(fname))
f = open(fname, 'wb')
keys = [
'Date', 'ut', 'utc', 'Alt', 'AltMid', 'Bm', 'BmMid', 'B0', 'AlphaN',
'AlphaS', 'BaltN', 'BaltS', 'LCAlt'
]
for k in keys:
if k == 'utc':
dtype = 'float64'
else:
dtype = 'float32'
pf.ArrayToFile(Mirror[k], dtype, f)
f.close()
#change permissions
os.system('chmod 666 ' + fname)
def ReadData():
'''
Read in the downloaded ASCII file
'''
fname = Globals.MessPath + 'FIPS/ANN/FIPSProtonClass.dat'
return pf.ReadASCIIData(fname, Header=True)
def ReadPAD(Date,path,SpecType):
'''
Read a PAD file
'''
#get the file name
fname = path + '{:08d}/'.format(Date) + SpecType + '.bin'
#check it exists
if not os.path.isfile(fname):
print('File not found')
return None
#read the data
f = open(fname,'rb')
out = {}
out['Date'] = pf.ArrayFromFile('int32',f)
out['ut'] = pf.ArrayFromFile('float32',f)
out['utc'] = pf.ArrayFromFile('float64',f)
out['Emin'] = pf.ArrayFromFile('float32',f)
out['Emax'] = pf.ArrayFromFile('float32',f)
out['Alpha'] = pf.ArrayFromFile('float32',f)
out['Flux'] = pf.ArrayFromFile('float32',f)
f.close()
return out
def SavePAD(Date, path, spec, Overwrite=False):
'''
Save pitch angle distribution data
'''
#create the output path
outpath = path + '{:08d}/'.format(Date)
if not os.path.isdir(outpath):
os.system('mkdir -pv ' + outpath)
os.system('chmod 777 ' + outpath)
#create a list of spectra
keys = list(spec.keys())
#loop through and save each one
for k in keys:
tmp = spec[k]
fname = outpath + k + '.bin'
if os.path.isfile(fname) and not Overwrite:
continue
print('saving file: {:s}'.format(fname))
f = open(fname, 'wb')
pf.ArrayToFile(tmp['Date'], 'int32', f)
pf.ArrayToFile(tmp['ut'], 'float32', f)
pf.ArrayToFile(tmp['utc'], 'float64', f)
pf.ArrayToFile(tmp['Emin'], 'float32', f)
pf.ArrayToFile(tmp['Emax'], 'float32', f)
pf.ArrayToFile(tmp['Alpha'], 'float32', f)
pf.ArrayToFile(tmp['Flux'], 'float32', f)
f.close()
#change permissions
os.system('chmod 666 ' + fname)
def _UpdateDataIndex(idx, fname):
'''
Updates the data index file.
Input:
idx: numpy.recarray containing the file names.
'''
pf.WriteASCIIData(fname, idx)
def _UpdateDataIndex(idx, L):
'''
Updates the data index file.
Input:
idx: numpy.recarray containing the file names.
'''
fname = Globals.DataPath + 'Pos/PosIndex-L{:01d}.dat'.format(L)
pf.WriteASCIIData(fname, idx)
def _UpdateDataIndex(idx,sc='a',Inst='hope',L='l3.moments'):
'''
Updates the data index file.
Input:
idx: numpy.recarray containing the file names.
'''
fname = Globals.DataPath+'ECT/{:s}.{:s}.{:s}.dat'.format(Inst,L,sc)
pf.WriteASCIIData(fname,idx)
def ReadSolarRotations():
'''
Read the file containing the solar rotations.
'''
path = Globals.OutputPath + 'Sun/'
fname = path + '/SunRotations.dat'
return pf.ReadASCIIData(fname, dtype=dtype)
def _UpdateDataIndex(idx,sc='a'):
'''
Updates the data index file.
Input:
idx: numpy.recarray containing the file names.
'''
fname = Globals.DataPath+'MagEph/{:s}.dat'.format(sc)
pf.WriteASCIIData(fname,idx)
def ReadMatrix(Name='P-3-m-2'):
'''
Read one of the test matrices.
'''
fname = Globals.MatrixPath + '{:s}.bin'.format(Name)
f = open(fname, 'rb')
P = pf.ArrayFromFile('float32', f)
f.close()
return P
def _UpdateDataIndex(idx):
'''
Updates the data index file.
Input:
idx: numpy.recarray containing the file names, time resolution
and dates for all data files
'''
fname = Globals.DataPath + 'DataIndex.dat'
pf.WriteASCIIData(fname, idx)
def _ReadMSHCrossings():
'''
Reads the file contining the list of crossings of MESSENGER through
the magnetosheath.
Returns:
numpy.recarray
'''
fname = Globals.ModulePath + '__data/MSHCrossings.dat'
dtype = [('Date0', 'int32'), ('ut0', 'float32'), ('Date1', 'int32'),
('ut1', 'float32'), ('dir', 'U1')]
return pf.ReadASCIIData(fname, Header=False, dtype=dtype)
def _ReadMercurySpeed():
'''
Reads the file which stored Mercury's orbital speed from 20080101
to 20150431.
'''
fname = Globals.ModulePath + '/__data/MercurySpeed.dat'
data = pf.ReadASCIIData(fname,
Header=True,
dtype=[('Date', 'int32'), ('utc', 'float64'),
('v', 'float32')])
return data
def _GetDateFiles(date):
#split date
yr = date // 10000
mn = (date % 10000) // 100
dy = date % 100
#get the url to search
url = url0.format(yr, mn, dy)
#download it
os.system('wget --no-verbose ' + url + ' -O tmp.html')
#read it in
files = []
lines = pf.ReadASCIIFile('tmp.html')
for l in lines:
s = l.split("'")
for ss in s:
if ".F01.gz" in ss and not '<' in ss:
files.append(ss)
files = np.array(files)
#delete tmp file
os.system('rm tmp.html')
#get the fnames
fnames = []
for f in files:
s = f.split('/')
fnames.append(s[-1])
fnames = np.array(fnames)
#output files
ofiles = []
opath = datapath.format(yr)
if not os.path.isdir(opath):
os.system('mkdir -pv ' + opath)
for f in fnames:
o = opath + f
ofiles.append(o)
ofiles = np.array(ofiles)
n = ofiles.size
#download them
for i in range(0, n):
os.system('wget --no-verbose ' + 'http://data.carisma.ca' + files[i] +
' -O ' + ofiles[i])
def _ReadMET():
'''
Reads a data file containing the mission elapsed times (METs) at the
start of every date from 20080101 - 20150430.
Returns:
numpy.recarray
'''
fname = Globals.ModulePath + '__data/MessengerMET.dat'
dtype = [('Date', 'int32'), ('ut', 'float32'), ('MET', 'float64')]
data = pf.ReadASCIIData(fname, dtype=dtype)
return data
def SaveSpeed(Date0=19500101, Date1=20500101):
Dates = ListDates(Date0, Date1)
ut = np.zeros(Dates.size, dtype='float32')
s = Speed(Dates)
sdtype = [('Date', 'int32'), ('utc', 'float64'), ('v', 'float32')]
data = np.recarray(Dates.size, dtype=sdtype)
data.Date = Dates
data.utc = ContUT(data.Date, np.zeros(data.size))
data.v = s
fname = Globals.OutputPath + 'Mercury/MercurySpeed.dat'
pf.WriteASCIIData(fname, data)
def _ReadIndexFile(fname):
'''
This will be reading the index file for a single magnetometer
station.
'''
dtype = [('Date', 'int32'), ('Station', 'object'), ('Res', 'float32'),
('File', 'object'), ('SubDir', 'object')]
if os.path.isfile(fname):
data = pf.ReadASCIIData(fname, Header=True, dtype=dtype)
else:
data = np.recarray(0, dtype=dtype)
return data
def _ReadIMAGE1s(fname):
'''
this is for reading the files with the name 'SSS_yyyymmdd.txt'
'''
dtype0 = [ ('yr','int32'),
('mn','int32'),
('dy','int32'),
('hh','int32'),
('mm','int32'),
('ss','int32'),
('Bx','float64'),
('By','float64'),
('Bz','float64')]
dtype = [ ('Date','int32'),
('ut','float64'),
('Bx','float64'),
('By','float64'),
('Bz','float64')]
try:
data = pf.ReadASCIIData(fname,Header=False,dtype=dtype0)
except:
return np.recarray(0,dtype=dtype)
badx = data.Bx > 90000.0
bady = data.By > 90000.0
badz = data.Bz > 90000.0
bad = np.where(badx | bady | badz)[0]
data.Bx[bad] = np.nan
data.By[bad] = np.nan
data.Bz[bad] = np.nan
n = data.size
out = np.recarray(n,dtype=dtype)
out.Date = TT.DateJoin(data.yr,data.mn,data.dy)
out.ut = TT.HHMMtoDec(data.hh,data.mm,data.ss)
out.Bx = data.Bx
out.By = data.By
out.Bz = data.Bz
return out
def _ReadCarisma8Hz(fname):
'''
This format is different to the others
'''
dtype = [ ('Date','int32'),
('ut','float64'),
('Bx','float64'),
('By','float64'),
('Bz','float64')]
lines = pf.ReadASCIIFile(fname)[1:]
nrec = lines.size//9
n = 8*nrec
data = np.recarray(n,dtype=dtype)
dt = np.arange(8)*0.125/3600.0
for i in range(0,nrec):
i0 = i*8
i1 = (i+1)*8
l0 = i*9
s = lines[l0].split()
data.Date[i0:i1] = np.int32(s[0])
hh = np.int32(s[1][0:2])
mm = np.int32(s[1][2:4])
ss = np.int32(s[1][4:6])
data.ut[i0:i1] = (hh + mm/60.0 + ss/3600.0) + dt
for j in range(0,8):
s = np.float64(lines[l0 + 1 + j].split())
data.Bx[i0 + j] = s[0]
data.By[i0 + j] = s[1]
data.Bz[i0 + j] = s[2]
badx = data.Bx > 90000.0
bady = data.By > 90000.0
badz = data.Bz > 90000.0
bad = np.where(badx | bady | badz)[0]
data.Bx[bad] = np.nan
data.By[bad] = np.nan
data.Bz[bad] = np.nan
return data
def _ReadDataIndex():
'''
Reads an index file containing the file names, update dates and
resolutions.
'''
#define the datatype
dtype = [('FileName', 'object'), ('OldFileName', 'object'),
('UpdateDate', 'object'), ('Res', 'int32')]
#check if the index file exists
fname = Globals.DataPath + 'DataIndex.dat'
if not os.path.isfile(fname):
return np.recarray(0, dtype=dtype)
#if it does exist, then read it
data = pf.ReadASCIIData(fname, Header=True, dtype=dtype)
return data
def _ParseFTP():
'''
This routine will read the FTP index file looking for file names
and their associated update dates.
'''
#read the file in
fname = Globals.DataPath + 'tmp/index.html'
lines = pf.ReadASCIIFile(fname)
nl = np.size(lines)
#firstly, search for the lines which contain 'omni_min' or 'omni_5min'
use = np.zeros(nl, dtype='int')
for i in range(0, nl):
if 'omni_min' in lines[i]:
use[i] = 1
elif 'omni_5min' in lines[i]:
use[i] = 5
keep = np.where(use > 0)[0]
Res = use[keep]
lines = lines[keep]
nl = lines.size
#now to extract the FTP address, file name and update dates
UpdateDates = np.zeros(nl, dtype='object')
Addresses = np.zeros(nl, dtype='object')
FileNames = np.zeros(nl, dtype='object')
for i in range(0, nl):
#deal with date first
s = lines[i].split()
UpdateDates[i] = s[1]
#now let's get the ftp address
#s = lines[i].split('"')
Addresses[
i] = 'https://spdf.gsfc.nasa.gov/pub/data/omni/high_res_omni/' + s[
0]
#now the file name
#s = s[1].split('/')
FileNames[i] = s[0]
return FileNames, Addresses, UpdateDates, Res
def ReadPDSMAG(fname):
'''
This will read in the .TAB datafile and output a numpy.recarray
Inputs:
fname: file name and path
'''
pdsdtype = [ ('Year','int32'),
('DOY','int32'),
('Hour','int32'),
('Min','int32'),
('Sec','float32'),
('MET','float32'),
('Xmso','float32'),
('Ymso','float32'),
('Zmso','float32'),
('Bx','float32'),
('By','float32'),
('Bz','float32'),
('Loc','U2')]
data = pf.ReadASCIIData(fname,False,dtype=pdsdtype)
dtype = MagGlobals.dtypes['MSM']
n = data.size
out = np.recarray(n,dtype)
out.Date = np.array([TT.DayNotoDate(data.Year[i],data.DOY[i])[0] for i in range(0,n)])
out.ut = np.float32(data.Hour) + np.float32(data.Min)/60.0 + np.float32(data.Sec)/3600.0
out.utc = ContUT(out.Date,out.ut)
out.Xmso = data.Xmso/2440.0
out.Ymso = data.Ymso/2440.0
out.Zmso = data.Zmso/2440.0
out.Xmsm = data.Xmso/2440.0
out.Ymsm = data.Ymso/2440.0
out.Zmsm = data.Zmso/2440.0 - 0.196
out.Bx = data.Bx
out.By = data.By
out.Bz = data.Bz
out.Loc = GetRegion(out.Date,out.ut,out.utc,Verbose=False)
return out
def _ReadDataIndex(sc='a'):
'''
Reads index file containing a list of all of the dates with their
associated data file name (so that we can pick the version
automatically).
'''
#define the dtype
dtype = [('Date', 'int32'), ('FileName', 'object'), ('Version', 'int16')]
#find the file
fname = Globals.DataPath + 'MagEph/{:s}.dat'.format(sc)
#check it exists
if not os.path.isfile(fname):
return np.recarray(0, dtype=dtype)
#read the index file
data = pf.ReadASCIIData(fname, True, dtype=dtype)
return data