def _ReadIMAGEiaga(fname):
'''
Read the mess of a format that is iaga.
Name format: 'image_yyyymmdd.iaga' (10s data)
'''
#read the data
lines = pf.ReadASCIIFile(fname)
s = lines[0]
#output dict
out = {}
#split into records
rlen = np.int32(s[:4])
ss = np.array([s[i*rlen:(i+1)*rlen] for i in range(0,len(s)//rlen)])
nr = len(ss)
#get ID
ID = np.array([ss[i][12:15] for i in range(0,nr)])
IDu = np.unique(ID)
for I in IDu:
use = np.where(ID == I)[0]
out[I] = _iaga_station(ss[use])
return out
def _ReadStations(Network):
'''
Read the list of stations for a magnetometer network.
'''
fname = Globals.StationPath + Network + '.list'
dtype = [('Code', 'U4'), ('Station', 'object'), ('Network', 'object'),
('glat', 'float32'), ('glon', 'float32'), ('mlat', 'float32'),
('mlon', 'float32'), ('L', 'float32')]
data = pf.ReadASCIIFile(fname)
n = data.size
out = np.recarray(n, dtype=dtype)
for i in range(0, n):
s = data[i].split()
#the number of bits which are the name
ln = len(s) - 6
out[i].Code = s[0]
out[i].Station = ' '.join(s[1:ln + 1])
out[i].glat = np.float32(s[1 + ln])
out[i].glon = np.float32(s[2 + ln])
out[i].mlat = np.float32(s[3 + ln])
out[i].mlon = np.float32(s[4 + ln])
out[i].L = np.float32(s[5 + ln])
out.Network[:] = Network
return out
def _ReadCarisma1Hz(fname):
'''
This will read the extracted 1 Hz Carisma data files with the file
name format 'yyyymmddSSSS.F01'.
'''
dtype = [ ('Date','int32'),
('ut','float64'),
('Bx','float64'),
('By','float64'),
('Bz','float64')]
dtype0 = [ ('dt','object'),
('Bx','float64'),
('By','float64'),
('Bz','float64'),
('nothing','object')]
try:
data = pf.ReadASCIIData(fname,Header=True,dtype=dtype0)
n = data.size
except:
lines = pf.ReadASCIIFile(fname)[1:]
n = lines.size
data = np.recarray(n,dtype=dtype0)
data.dt = np.array([l[:14] for l in lines])
for i in range(0,n):
try:
data.Bx[i] = np.float64(l[14:24])
data.By[i] = np.float64(l[24:34])
data.Bz[i] = np.float64(l[34:44])
except:
data.Bx[i] = np.nan
data.By[i] = np.nan
data.Bz[i] = np.nan
out = np.recarray(n,dtype=dtype)
out.Bx = data.Bx
out.By = data.By
out.Bz = data.Bz
out.Date = np.array([np.int32(s[:8]) for s in data.dt])
for i in range(0,n):
h = np.int32(data.dt[i][8:10])
m = np.int32(data.dt[i][10:12])
s = np.int32(data.dt[i][12:14])
out.ut[i] = h + m/60 + s/3600.0
badx = out.Bx > 90000.0
bady = out.By > 90000.0
badz = out.Bz > 90000.0
bad = np.where(badx | bady | badz)[0]
out.Bx[bad] = np.nan
out.By[bad] = np.nan
out.Bz[bad] = np.nan
return out
def _ReadCanopus(fname):
'''
This will read the extracted 0.2 Hz Canopus data files with the file
name format 'yyyymmddSSSS.MAG'.
'''
dtype = [ ('Date','int32'),
('ut','float64'),
('Bx','float64'),
('By','float64'),
('Bz','float64')]
dtype0 = [ ('dt','object'),
('Bx','float64'),
('By','float64'),
('Bz','float64'),
('nothing','object')]
lines = pf.ReadASCIIFile(fname)
n = lines.size
for i in range(0,n):
if lines[i][0] != '#':
break
lines = lines[i+1:]
try:
data = pf.ReadASCIIData(lines.tolist(),Header=False,dtype=dtype0)
except:
return np.recarray(0,dtype=dtype)
n = data.size
out = np.recarray(n,dtype=dtype)
out.Bx = data.Bx
out.By = data.By
out.Bz = data.Bz
out.Date = np.array([np.int32(s[:8]) for s in data.dt])
for i in range(0,n):
h = np.int32(data.dt[i][8:10])
m = np.int32(data.dt[i][10:12])
s = np.int32(data.dt[i][12:14])
out.ut[i] = h + m/60 + s/3600.0
badx = out.Bx > 90000.0
bady = out.By > 90000.0
badz = out.Bz > 90000.0
bad = np.where(badx | bady | badz)[0]
out.Bx[bad] = np.nan
out.By[bad] = np.nan
out.Bz[bad] = np.nan
return out
def _ReadIAGA2002(fname):
'''
This function will read the IAGA 2002 data format used by
INTERMAGNET.
'''
dtype = [('Date', 'int32'), ('ut', 'float64'), ('Bx', 'float64'),
('By', 'float64'), ('Bz', 'float64')]
dtype0 = [('Date', 'object'), ('ut', 'object'), ('doy', 'int32'),
('Bx', 'float64'), ('By', 'float64'), ('Bz', 'float64'),
('Bm', 'float64')]
lines = pf.ReadASCIIFile(fname)
n = lines.size
for i in range(0, n):
if lines[i][0:4] == 'DATE':
break
lines = lines[i + 1:]
try:
data = pf.ReadASCIIData(lines.tolist(), Header=False, dtype=dtype0)
except:
return np.recarray(0, dtype=dtype)
n = data.size
out = np.recarray(n, dtype=dtype)
out.Bx = data.Bx
out.By = data.By
out.Bz = data.Bz
out.Date = np.array([np.int32(D.replace('-', '')) for D in data.Date])
for i in range(0, n):
h = np.int32(data.ut[i][0:2])
m = np.int32(data.ut[i][3:5])
s = np.float32(data.ut[i][6:])
out.ut[i] = h + m / 60 + s / 3600.0
badx = out.Bx > 90000.0
bady = out.By > 90000.0
badz = out.Bz > 90000.0
bad = np.where(badx | bady | badz)[0]
out.Bx[bad] = np.nan
out.By[bad] = np.nan
out.Bz[bad] = np.nan
return out
def _GetDateFiles(date):
#split date
yr = date // 10000
mn = (date % 10000) // 100
dy = date % 100
#get the url to search
url = url0.format(yr, mn, dy)
#download it
os.system('wget --no-verbose ' + url + ' -O tmp.html')
#read it in
files = []
lines = pf.ReadASCIIFile('tmp.html')
for l in lines:
s = l.split("'")
for ss in s:
if ".F01.gz" in ss and not '<' in ss:
files.append(ss)
files = np.array(files)
#delete tmp file
os.system('rm tmp.html')
#get the fnames
fnames = []
for f in files:
s = f.split('/')
fnames.append(s[-1])
fnames = np.array(fnames)
#output files
ofiles = []
opath = datapath.format(yr)
if not os.path.isdir(opath):
os.system('mkdir -pv ' + opath)
for f in fnames:
o = opath + f
ofiles.append(o)
ofiles = np.array(ofiles)
n = ofiles.size
#download them
for i in range(0, n):
os.system('wget --no-verbose ' + 'http://data.carisma.ca' + files[i] +
' -O ' + ofiles[i])
def _ReadCarisma8Hz(fname):
'''
This format is different to the others
'''
dtype = [ ('Date','int32'),
('ut','float64'),
('Bx','float64'),
('By','float64'),
('Bz','float64')]
lines = pf.ReadASCIIFile(fname)[1:]
nrec = lines.size//9
n = 8*nrec
data = np.recarray(n,dtype=dtype)
dt = np.arange(8)*0.125/3600.0
for i in range(0,nrec):
i0 = i*8
i1 = (i+1)*8
l0 = i*9
s = lines[l0].split()
data.Date[i0:i1] = np.int32(s[0])
hh = np.int32(s[1][0:2])
mm = np.int32(s[1][2:4])
ss = np.int32(s[1][4:6])
data.ut[i0:i1] = (hh + mm/60.0 + ss/3600.0) + dt
for j in range(0,8):
s = np.float64(lines[l0 + 1 + j].split())
data.Bx[i0 + j] = s[0]
data.By[i0 + j] = s[1]
data.Bz[i0 + j] = s[2]
badx = data.Bx > 90000.0
bady = data.By > 90000.0
badz = data.Bz > 90000.0
bad = np.where(badx | bady | badz)[0]
data.Bx[bad] = np.nan
data.By[bad] = np.nan
data.Bz[bad] = np.nan
return data
def _ParseFTP():
'''
This routine will read the FTP index file looking for file names
and their associated update dates.
'''
#read the file in
fname = Globals.DataPath + 'tmp/index.html'
lines = pf.ReadASCIIFile(fname)
nl = np.size(lines)
#firstly, search for the lines which contain 'omni_min' or 'omni_5min'
use = np.zeros(nl, dtype='int')
for i in range(0, nl):
if 'omni_min' in lines[i]:
use[i] = 1
elif 'omni_5min' in lines[i]:
use[i] = 5
keep = np.where(use > 0)[0]
Res = use[keep]
lines = lines[keep]
nl = lines.size
#now to extract the FTP address, file name and update dates
UpdateDates = np.zeros(nl, dtype='object')
Addresses = np.zeros(nl, dtype='object')
FileNames = np.zeros(nl, dtype='object')
for i in range(0, nl):
#deal with date first
s = lines[i].split()
UpdateDates[i] = s[1]
#now let's get the ftp address
#s = lines[i].split('"')
Addresses[
i] = 'https://spdf.gsfc.nasa.gov/pub/data/omni/high_res_omni/' + s[
0]
#now the file name
#s = s[1].split('/')
FileNames[i] = s[0]
return FileNames, Addresses, UpdateDates, Res
def _ConvertSolarFlux():
'''
Convert the ascii file from OMNI to a binary file
'''
#get the ascii file name
fname0 = Globals.DataPath + 'F107.lst'
#read the ascii file
lines = pf.ReadASCIIFile(fname0)
#remove the header
for i in range(0, lines.size):
if lines[i][:4] == 'YEAR':
lines = lines[i + 1:]
break
#remove the crap at the end
for i in range(0, lines.size):
if lines[i][0] == '<':
lines = lines[:i]
break
n = lines.size
#split into separate strings
s = np.array([l.split() for l in lines], dtype='object')
#create the output array
data = np.recarray(n, dtype=Globals.fdtype)
yr = np.int32(s[:, 0])
doy = np.int32(s[:, 1])
data.Date = TT.DayNotoDate(yr, doy)
data.ut = np.float32(s[:, 2])
data.utc = TT.ContUT(data.Date, data.ut)
data.F10_7 = np.float32(s[:, 3])
#save the new file
fname1 = Globals.DataPath + 'F107.bin'
RT.SaveRecarray(data, fname1)
def _ReadOrbits():
'''
Reads the Orbits.dat file to return a numpy.ndarray containing the
start and end times of the MESSENGER orbits around Mercury.
'''
dtype = [('Orbit','int32'),('Date','int32',(2,)),('ut','float32',(2,))]
fname = Globals.ModulePath+'__data/Orbits.dat'
lines = pf.ReadASCIIFile(fname)
nl = np.size(lines)
data = np.recarray(nl,dtype=dtype)
for i in range(0,nl):
s = lines[i].split()
data[i].Orbit = np.int32(s[0])
data[i].Date = np.array([s[1],s[3]])
data[i].ut = np.array([s[2],s[4]])
return data