def _ReadIMAGEiaga(fname): ''' Read the mess of a format that is iaga. Name format: 'image_yyyymmdd.iaga' (10s data) ''' #read the data lines = pf.ReadASCIIFile(fname) s = lines[0] #output dict out = {} #split into records rlen = np.int32(s[:4]) ss = np.array([s[i*rlen:(i+1)*rlen] for i in range(0,len(s)//rlen)]) nr = len(ss) #get ID ID = np.array([ss[i][12:15] for i in range(0,nr)]) IDu = np.unique(ID) for I in IDu: use = np.where(ID == I)[0] out[I] = _iaga_station(ss[use]) return out
def _ReadStations(Network): ''' Read the list of stations for a magnetometer network. ''' fname = Globals.StationPath + Network + '.list' dtype = [('Code', 'U4'), ('Station', 'object'), ('Network', 'object'), ('glat', 'float32'), ('glon', 'float32'), ('mlat', 'float32'), ('mlon', 'float32'), ('L', 'float32')] data = pf.ReadASCIIFile(fname) n = data.size out = np.recarray(n, dtype=dtype) for i in range(0, n): s = data[i].split() #the number of bits which are the name ln = len(s) - 6 out[i].Code = s[0] out[i].Station = ' '.join(s[1:ln + 1]) out[i].glat = np.float32(s[1 + ln]) out[i].glon = np.float32(s[2 + ln]) out[i].mlat = np.float32(s[3 + ln]) out[i].mlon = np.float32(s[4 + ln]) out[i].L = np.float32(s[5 + ln]) out.Network[:] = Network return out
def _ReadCarisma1Hz(fname): ''' This will read the extracted 1 Hz Carisma data files with the file name format 'yyyymmddSSSS.F01'. ''' dtype = [ ('Date','int32'), ('ut','float64'), ('Bx','float64'), ('By','float64'), ('Bz','float64')] dtype0 = [ ('dt','object'), ('Bx','float64'), ('By','float64'), ('Bz','float64'), ('nothing','object')] try: data = pf.ReadASCIIData(fname,Header=True,dtype=dtype0) n = data.size except: lines = pf.ReadASCIIFile(fname)[1:] n = lines.size data = np.recarray(n,dtype=dtype0) data.dt = np.array([l[:14] for l in lines]) for i in range(0,n): try: data.Bx[i] = np.float64(l[14:24]) data.By[i] = np.float64(l[24:34]) data.Bz[i] = np.float64(l[34:44]) except: data.Bx[i] = np.nan data.By[i] = np.nan data.Bz[i] = np.nan out = np.recarray(n,dtype=dtype) out.Bx = data.Bx out.By = data.By out.Bz = data.Bz out.Date = np.array([np.int32(s[:8]) for s in data.dt]) for i in range(0,n): h = np.int32(data.dt[i][8:10]) m = np.int32(data.dt[i][10:12]) s = np.int32(data.dt[i][12:14]) out.ut[i] = h + m/60 + s/3600.0 badx = out.Bx > 90000.0 bady = out.By > 90000.0 badz = out.Bz > 90000.0 bad = np.where(badx | bady | badz)[0] out.Bx[bad] = np.nan out.By[bad] = np.nan out.Bz[bad] = np.nan return out
def _ReadCanopus(fname): ''' This will read the extracted 0.2 Hz Canopus data files with the file name format 'yyyymmddSSSS.MAG'. ''' dtype = [ ('Date','int32'), ('ut','float64'), ('Bx','float64'), ('By','float64'), ('Bz','float64')] dtype0 = [ ('dt','object'), ('Bx','float64'), ('By','float64'), ('Bz','float64'), ('nothing','object')] lines = pf.ReadASCIIFile(fname) n = lines.size for i in range(0,n): if lines[i][0] != '#': break lines = lines[i+1:] try: data = pf.ReadASCIIData(lines.tolist(),Header=False,dtype=dtype0) except: return np.recarray(0,dtype=dtype) n = data.size out = np.recarray(n,dtype=dtype) out.Bx = data.Bx out.By = data.By out.Bz = data.Bz out.Date = np.array([np.int32(s[:8]) for s in data.dt]) for i in range(0,n): h = np.int32(data.dt[i][8:10]) m = np.int32(data.dt[i][10:12]) s = np.int32(data.dt[i][12:14]) out.ut[i] = h + m/60 + s/3600.0 badx = out.Bx > 90000.0 bady = out.By > 90000.0 badz = out.Bz > 90000.0 bad = np.where(badx | bady | badz)[0] out.Bx[bad] = np.nan out.By[bad] = np.nan out.Bz[bad] = np.nan return out
def _ReadIAGA2002(fname): ''' This function will read the IAGA 2002 data format used by INTERMAGNET. ''' dtype = [('Date', 'int32'), ('ut', 'float64'), ('Bx', 'float64'), ('By', 'float64'), ('Bz', 'float64')] dtype0 = [('Date', 'object'), ('ut', 'object'), ('doy', 'int32'), ('Bx', 'float64'), ('By', 'float64'), ('Bz', 'float64'), ('Bm', 'float64')] lines = pf.ReadASCIIFile(fname) n = lines.size for i in range(0, n): if lines[i][0:4] == 'DATE': break lines = lines[i + 1:] try: data = pf.ReadASCIIData(lines.tolist(), Header=False, dtype=dtype0) except: return np.recarray(0, dtype=dtype) n = data.size out = np.recarray(n, dtype=dtype) out.Bx = data.Bx out.By = data.By out.Bz = data.Bz out.Date = np.array([np.int32(D.replace('-', '')) for D in data.Date]) for i in range(0, n): h = np.int32(data.ut[i][0:2]) m = np.int32(data.ut[i][3:5]) s = np.float32(data.ut[i][6:]) out.ut[i] = h + m / 60 + s / 3600.0 badx = out.Bx > 90000.0 bady = out.By > 90000.0 badz = out.Bz > 90000.0 bad = np.where(badx | bady | badz)[0] out.Bx[bad] = np.nan out.By[bad] = np.nan out.Bz[bad] = np.nan return out
def _GetDateFiles(date): #split date yr = date // 10000 mn = (date % 10000) // 100 dy = date % 100 #get the url to search url = url0.format(yr, mn, dy) #download it os.system('wget --no-verbose ' + url + ' -O tmp.html') #read it in files = [] lines = pf.ReadASCIIFile('tmp.html') for l in lines: s = l.split("'") for ss in s: if ".F01.gz" in ss and not '<' in ss: files.append(ss) files = np.array(files) #delete tmp file os.system('rm tmp.html') #get the fnames fnames = [] for f in files: s = f.split('/') fnames.append(s[-1]) fnames = np.array(fnames) #output files ofiles = [] opath = datapath.format(yr) if not os.path.isdir(opath): os.system('mkdir -pv ' + opath) for f in fnames: o = opath + f ofiles.append(o) ofiles = np.array(ofiles) n = ofiles.size #download them for i in range(0, n): os.system('wget --no-verbose ' + 'http://data.carisma.ca' + files[i] + ' -O ' + ofiles[i])
def _ReadCarisma8Hz(fname): ''' This format is different to the others ''' dtype = [ ('Date','int32'), ('ut','float64'), ('Bx','float64'), ('By','float64'), ('Bz','float64')] lines = pf.ReadASCIIFile(fname)[1:] nrec = lines.size//9 n = 8*nrec data = np.recarray(n,dtype=dtype) dt = np.arange(8)*0.125/3600.0 for i in range(0,nrec): i0 = i*8 i1 = (i+1)*8 l0 = i*9 s = lines[l0].split() data.Date[i0:i1] = np.int32(s[0]) hh = np.int32(s[1][0:2]) mm = np.int32(s[1][2:4]) ss = np.int32(s[1][4:6]) data.ut[i0:i1] = (hh + mm/60.0 + ss/3600.0) + dt for j in range(0,8): s = np.float64(lines[l0 + 1 + j].split()) data.Bx[i0 + j] = s[0] data.By[i0 + j] = s[1] data.Bz[i0 + j] = s[2] badx = data.Bx > 90000.0 bady = data.By > 90000.0 badz = data.Bz > 90000.0 bad = np.where(badx | bady | badz)[0] data.Bx[bad] = np.nan data.By[bad] = np.nan data.Bz[bad] = np.nan return data
def _ParseFTP(): ''' This routine will read the FTP index file looking for file names and their associated update dates. ''' #read the file in fname = Globals.DataPath + 'tmp/index.html' lines = pf.ReadASCIIFile(fname) nl = np.size(lines) #firstly, search for the lines which contain 'omni_min' or 'omni_5min' use = np.zeros(nl, dtype='int') for i in range(0, nl): if 'omni_min' in lines[i]: use[i] = 1 elif 'omni_5min' in lines[i]: use[i] = 5 keep = np.where(use > 0)[0] Res = use[keep] lines = lines[keep] nl = lines.size #now to extract the FTP address, file name and update dates UpdateDates = np.zeros(nl, dtype='object') Addresses = np.zeros(nl, dtype='object') FileNames = np.zeros(nl, dtype='object') for i in range(0, nl): #deal with date first s = lines[i].split() UpdateDates[i] = s[1] #now let's get the ftp address #s = lines[i].split('"') Addresses[ i] = 'https://spdf.gsfc.nasa.gov/pub/data/omni/high_res_omni/' + s[ 0] #now the file name #s = s[1].split('/') FileNames[i] = s[0] return FileNames, Addresses, UpdateDates, Res
def _ConvertSolarFlux(): ''' Convert the ascii file from OMNI to a binary file ''' #get the ascii file name fname0 = Globals.DataPath + 'F107.lst' #read the ascii file lines = pf.ReadASCIIFile(fname0) #remove the header for i in range(0, lines.size): if lines[i][:4] == 'YEAR': lines = lines[i + 1:] break #remove the crap at the end for i in range(0, lines.size): if lines[i][0] == '<': lines = lines[:i] break n = lines.size #split into separate strings s = np.array([l.split() for l in lines], dtype='object') #create the output array data = np.recarray(n, dtype=Globals.fdtype) yr = np.int32(s[:, 0]) doy = np.int32(s[:, 1]) data.Date = TT.DayNotoDate(yr, doy) data.ut = np.float32(s[:, 2]) data.utc = TT.ContUT(data.Date, data.ut) data.F10_7 = np.float32(s[:, 3]) #save the new file fname1 = Globals.DataPath + 'F107.bin' RT.SaveRecarray(data, fname1)
def _ReadOrbits(): ''' Reads the Orbits.dat file to return a numpy.ndarray containing the start and end times of the MESSENGER orbits around Mercury. ''' dtype = [('Orbit','int32'),('Date','int32',(2,)),('ut','float32',(2,))] fname = Globals.ModulePath+'__data/Orbits.dat' lines = pf.ReadASCIIFile(fname) nl = np.size(lines) data = np.recarray(nl,dtype=dtype) for i in range(0,nl): s = lines[i].split() data[i].Orbit = np.int32(s[0]) data[i].Date = np.array([s[1],s[3]]) data[i].ut = np.array([s[2],s[4]]) return data