def write2HDF(data, h5fn, wl): obstimes = data.time.values.astype(datetime) ts = gu.datetime2posix(obstimes) az = data.az[1] el = data.el[1] images = data[wl].values N = az.shape[0] lat0 = data.lat lon0 = data.lon alt0 = data.alt_m try: f = h5py.File(h5fn, 'w') d = f.create_group('DASC') d.attrs[u'converted'] = datetime.now().strftime('%Y-%m-%d') d.attrs[u'wavelength'] = '{}'.format(wl) d.attrs[u'image resolution'] = '{}'.format(N) d.attrs[u'PKR camera lon'] = '{}'.format(lon0) d.attrs[u'PKR camera lat'] = '{}'.format(lat0) d.attrs[u'PKR camera alt'] = '{}'.format(alt0) h5time = d.create_dataset('time', data=ts) h5time.attrs[u'time format'] = 'time format in POSIX time' d.create_dataset('az', data=az) d.create_dataset('el', data=el) h5img = d.create_dataset('img', data=images, compression=9) h5img.chunks h5img.attrs[u'Coordinates'] = 'Ntimes x Naz x Nel' # close file f.close() except Exception as e: raise (e)
def correctSampling(t,y,Ts=1): if isinstance(t[0], datetime.datetime): ts = gnssUtils.datetime2posix(t) else: ts = t time_series = np.arange(int(ts[0]),int(ts[-1])+1,Ts) y_out = np.nan*np.ones((time_series.shape[0])) idt = np.where(np.isin(time_series,ts))[0] y_out[idt] = y return time_series, y_out
def _alignTimes(tlist, teclist, polylist, residuallist, fs): tmin = [] tmax = [] for i in range(len(tlist)): tmin.append(tlist[i].min()) tmax.append(tlist[i].max()) tstart = max(gnssUtils.datetime2posix(tmin)) tend = min(gnssUtils.datetime2posix(tmax)) t = [] tec2 = [] poly2 = [] res2 = [] for i in range(len(teclist)): tt, tec1 = correctSampling(tlist[i], teclist[i], fs=fs) tt, poly1 = correctSampling(tlist[i], polylist[i], fs=fs) tt, res1 = correctSampling(tlist[i], residuallist[i], fs=fs) tt = np.array(tt) idt = np.where((tt>=tstart) & (tt<=tend))[0] t.append(tt[idt]) tec2.append(tec1[idt]) poly2.append(poly1[idt]) res2.append(res1[idt]) return t, tec2, poly2, res2
def feed2hdf(t, tec, lat=[], lon=[], fn='/home/smrak/Documents/eclipse/surav/233.h5'): if isinstance(t[0], datetime): t = gnssUtils.datetime2posix(t) f = h5py.File(fn, 'w') gr = f.create_group('data') gr.create_dataset('times', data=t) gr.create_dataset('dtec', data=tec) gr.attrs['lon'] = lon gr.attrs['lat'] = lat f.close() return
def writeInterpolated2HDF(t, lon, lat, h, images, h5fn, lon0=0, lat0=0, alt0=0, N=None, wl=0): if isinstance(t[0], datetime): t = gu.datetime2posix(t) if N == 0 or N is None: N = lon.shape[0] try: f = h5py.File(h5fn, 'w') d = f.create_group('DASC') d.attrs[u'converted'] = datetime.now().strftime('%Y-%m-%d') d.attrs[u'wavelength'] = '{}'.format(wl) d.attrs[u'image resolution'] = '{}'.format(N) d.attrs[u'PKR camera lon'] = '{}'.format(lon0) d.attrs[u'PKR camera lat'] = '{}'.format(lat0) d.attrs[u'PKR camera alt'] = '{}'.format(alt0) h5time = d.create_dataset('time', data=t) h5time.attrs[u'time format'] = 'time format in POSIX time' d.create_dataset('lon', data=lon) d.create_dataset('lat', data=lat) H = d.create_dataset('alt', data=h) H.attrs[u'Mapping height'] = '{}'.format(h) h5img = d.create_dataset('img', data=images, compression=9) h5img.chunks h5img.attrs[u'Coordinates'] = 'Ntimes x Nlon x Nlat' # Close file f.close() except Exception as e: raise (e)
def save2HDF(t, lon, lat, images, h5fn): if isinstance(t[0], datetime): t = gu.datetime2posix(t) try: f = h5py.File(h5fn, 'w') d = f.create_group('GPSTEC') d.attrs[u'converted'] = datetime.now().strftime('%Y-%m-%d %H:%M') h5time = d.create_dataset('time', data=t) h5time.attrs[u'time format'] = 'time format in POSIX time' d.create_dataset('lon', data=lon) d.create_dataset('lat', data=lat) h5img = d.create_dataset('im', data=images, compression='gzip', compression_opts=9) h5img.chunks # Close file f.close() except Exception as e: if 'f' in locals(): f.close() raise (e)
tmp = np.rot90(im[i], rot) title = 'DASC: {} UT'.format(t[i]) imgname = datetime.strftime(t[i], '%H%M%S') + '.png' tmp[tmp <= 200] = np.nan fig = asiplot.plotIMpolar(xgrid, ygrid, tmp, title=title, figure=True, clim=[300, 600], cmap='Greys_r', norm_gamma=0.5, savefn=savepolar + imgname) if w2f: ts = gu.datetime2posix(t) try: f = h5py.File(h5fn, 'w') d = f.create_group('DASC') d.attrs[u'converted'] = datetime.now().strftime('%Y-%m-%d') d.attrs[u'wavelength'] = '{}'.format(wl) d.attrs[u'image resolution'] = '{}'.format(N) d.attrs[u'PKR camera lon'] = '{}'.format(lon) d.attrs[u'PKR camera lat'] = '{}'.format(lat) d.attrs[u'PKR camera alt'] = '{}'.format(alt) h5time = d.create_dataset('time', data=ts) h5time.attrs[u'time format'] = 'time format in POSIX time' d.create_dataset('xgrid', data=xgrid) d.create_dataset('ygrid', data=ygrid) h5img = d.create_dataset('img', data=im, compression=9) h5img.chunks
print(e) # if P.log: # LOG.write(str(e) + '\n') # pass # else: # print (e) except Exception as e: print(e) # if P.log: # with open(logfn, 'a') as LOG: # LOG.write(str(e) + '\n') # LOG.close() # else: # print (e) th5 = gu.datetime2posix(t.astype(datetime)) # Dealing with duplicate file names if os.path.exists(savefn): head = os.path.splitext(savefn)[0] c = 0 while os.path.exists(savefn): try: c = int(os.path.splitext(savefn)[0].split('_')[-1]) c += 1 except: c += 1 savefn = head + '_' + str(c) + '.h5' # putting the output file togather if P.log:
def getKeogram(filename, skip=1, X=40, lim=[-110, -70], direction='lat', timelim=[], im_filter=1, fillPixel_iter=0, integrate=2, spectogram=0, totality=0): """ Direction lat: Fixed LAT, look along lon """ data = h5py.File(filename, 'r') time = data['data/time'][::skip] xgrid = data['data/xgrid'].value ygrid = data['data/ygrid'].value im = data['data/im'][::skip][:][:] #--------------------------------------------------------------------------# if len(timelim) > 0: tl_posix = gu.datetime2posix(timelim) idt = np.where((time >= tl_posix[0]) & (time <= tl_posix[1]))[0] im = im[idt, :, :] time = time[idt] #--------------------------------------------------------------------------# if direction == 'lon': idy = np.where((ygrid >= lim[0]) & (ygrid <= lim[1]))[0] if integrate <= 1: idx = abs(xgrid - X).argmin() keogram = im[:, idx, idy] elif integrate > 1: idx = abs(xgrid - X).argmin() idx = np.array( [idx - int(integrate / 2), idx, idx + int(integrate / 2)]) imstack = im[:, idx, idy[0]:idy[-1]] keogram = np.nan * np.zeros((imstack.shape[0], imstack.shape[2])) for i in range(imstack.shape[0]): for j in range(imstack.shape[2]): N = sum(np.isfinite(imstack[i, :, j])) if N > 0: keogram[i, j] = sum(np.nan_to_num(imstack[i, :, j])) / N Y = ygrid[idy] elif direction == 'lat': idx = np.where((xgrid >= lim[0]) & (xgrid <= lim[1]))[0] if integrate <= 1: idy = abs(ygrid - X).argmin() keogram = im[:, idx, idy] elif integrate > 1: idy = abs(ygrid - X).argmin() idy = np.array( [idy - int(integrate / 2), idy, idy + int(integrate / 2)]) imstack = im[:, idx[0]:idx[-1], idy] keogram = np.nan * np.zeros((imstack.shape[0], imstack.shape[1])) for i in range(imstack.shape[0]): for j in range(imstack.shape[1]): N = sum(np.isfinite(imstack[i, j, :])) if N > 0: keogram[i, j] = sum(np.nan_to_num(imstack[i, j, :])) / N Y = xgrid[idx] elif direction == 'custom': if totality: totality_path = h5py.File( '/home/smrak/Documents/eclipse/totality.h5', 'r') Xt = totality_path['path/center_lon'].value Yt = totality_path['path/center_lat'].value - 1 Xt, Yt = interpolateTotality(Xt, Yt, lon0=-150, lon1=-50, order=5, resolution=0.3) else: Xt = np.array([-100, -93.75, -87.5, -81.2]) Yt = np.array([44, 39.5, 35, 30.5]) Xt = np.array([-95, -90]) Yt = np.array([44, 35]) Xt, Yt = interpolateTotality(Xt, Yt, lon0=Xt[0], lon1=Xt[-1], order=1, resolution=0.02) # plt.plot(Xt,Yt) idx = np.where((xgrid >= lim[0]) & (xgrid <= lim[1]))[0] idy = [] keogram = np.nan * np.ones((time.shape[0], xgrid[idx].shape[0])) for x in xgrid[idx]: idX = abs(x - Xt).argmin() Yt_approx = Yt[idX] tmp = abs(ygrid - Yt_approx).argmin() idy.append(tmp) idy = np.array(idy) if integrate <= 1: keogram = im[:, idx, idy] else: imkeogram = np.nan * np.ones( (time.shape[0], xgrid[idx].shape[0], integrate + 1)) intrange = np.arange(-int(integrate / 2), int(integrate / 2) + 1) for i in range(intrange.shape[0]): imkeogram[:, :, i] = im[:, idx, idy + intrange[i]] for i in range(imkeogram.shape[0]): for j in range(imkeogram.shape[1]): N = sum(np.isfinite(imkeogram[i, j, :])) if N > 0: keogram[i, j] = sum(np.nan_to_num(imkeogram[i, j, :])) / N Y = xgrid[idx] dt = [datetime.datetime.utcfromtimestamp(t) for t in time] if fillPixel_iter > 0: keogram = fillPixels(keogram, N=fillPixel_iter) if im_filter: keogram = ndimage.median_filter(keogram, 3) if spectogram: if direction == 'lat': n = Y.shape[0] d = round(ygrid[1] - ygrid[0], 1) * 111 * np.cos(np.deg2rad(X)) else: n = Y.shape[0] d = round(ygrid[1] - ygrid[0], 1) * 111 k = np.fft.fftfreq(n, d=d) k = 2 * np.pi * k[1:int((n - 1) / 2)] specto = np.nan * np.zeros((keogram.shape[0], k.shape[0])) # print (keogram.shape, Y.shape, specto.shape, n, k.shape) for j in range(keogram.shape[0]): x = np.nan_to_num(keogram[j]) Sx = np.abs(np.fft.fft(x)) Sx = Sx[1:int((n - 1) / 2)] specto[j, :] = Sx return [dt, keogram, Y], [k, specto] else: return [dt, keogram, Y]
tidim = plt.scatter(X[0], X[1], c=tec.T, s=8, cmap=cmap, transform=ccrs.PlateCarree()) else: tidim = plt.pcolormesh(xgrid,ygrid,tec.T,cmap=cmap,transform=ccrs.PlateCarree()) plt.clim(clim) # Fig utils cbar = plt.colorbar(mappable=tidim, #cax=cax, ticks=[clim[0], clim[0]/2, 0, clim[1]/2, clim[1]]) cbar.set_label('$\Delta$TEC [TECu]') if MASK: from dateutil import parser from pyGnss import gnssUtils dt = parser.parse(T) tposix = gnssUtils.datetime2posix([dt]) mask_x, mask_y, mask = gps2d.getEUVMask(tposix[0], EUVDIR='C:\\Users\\smrak\\Google Drive\BU\\Projects\\Eclipse2017\\data\\Drob\\HiResFull300\\') penumbra_levels = [0.2,1,40] levels = linspace(penumbra_levels[0],penumbra_levels[1],penumbra_levels[2]) lw = 1 plt.contour(mask_x, mask_y, mask.T, levels, colors='w', #cmap='Greys_r',#colors='w', linewidths=lw, transform=ccrs.PlateCarree()) if NEXRAD: if 'title' in vars(): title += '\n NEXRAD: ' + nqimage[7:-4] else: title = 'NEXRAD: ' + nqimage[7:-4] X, Y, Z = gps2d.returnNEXRAD(nexradfolder, downsample=16, darg=nqimage, RGB=RGB) if not RGB:
def main(F, el_mask = None, odir = None): global tsps D = h5py.File(F, 'r') el_mask = 30 maxjump = 1.6 + (np.sqrt(tsps) - 1) eps = 1 * np.sqrt(30/tsps) polynom_list = np.arange(0,20) el_mask_in = (el_mask - 10) if (el_mask - 10) >= 8 else 8 print ('Reading in receiver names') t0 = datetime.now() rxn_all = np.asanyarray([row[12].decode() for row in D['Data/Table Layout'][()]]) rxn = np.unique(rxn_all) print ('Read in {}.\nReading in satellite number'.format(datetime.now()-t0)) t0 = datetime.now() sv_unique = np.unique(np.asanyarray([row[13] for row in D['Data/Table Layout'][()]])) sv_index = np.arange(sv_unique.size) sv_list = {} for i, s in enumerate(sv_unique): sv_list[str(s)] = sv_index[i] print ('Read in {}.\nReading in observations times'.format(datetime.now()-t0)) obstimes = np.unique(np.asanyarray([datetime.utcfromtimestamp(row[9]) for row in D['Data/Table Layout'][()]])) obstimes_unix = gu.datetime2posix(obstimes) print ('Read in {}.\n'.format(datetime.now()-t0)) D.close() # Out-filename if odir is None: odir = os.path.split(F)[0] sfn = str(obstimes[0].year) + '_' + obstimes[0].strftime('%m%dT%H%M') + '-' + obstimes[-1].strftime('%m%dT%H%M') + '_' + 'madrigallos' + '_' + str(el_mask) +'el_' + str(tsps) + 's' + '_roti' savefn = os.path.join(odir, sfn + '.h5') # Duplicate file names if os.path.exists(savefn): head = os.path.splitext(savefn)[0] c = 0 while os.path.exists(savefn): try: c = int(os.path.splitext(savefn)[0].split('_')[-1]) c += 1 except: c += 1 savefn = head + '_' + str(c) + '.h5' logfn = os.path.splitext(savefn)[0] + '.log' LOG = open(logfn, 'w') LOG.close() print ('Init arrays') TEC = np.nan * np.zeros((obstimes.size, sv_unique.size, rxn.size), dtype=np.float16) DTEC = np.nan * np.zeros((obstimes.size, sv_unique.size, rxn.size), dtype=np.float16) ROTI = np.nan * np.zeros((obstimes.size, sv_unique.size, rxn.size), dtype=np.float16) AZ = np.nan * np.zeros((obstimes.size, sv_unique.size, rxn.size), dtype=np.float16) EL = np.nan * np.zeros((obstimes.size, sv_unique.size, rxn.size), dtype=np.float16) RXP = np.nan * np.zeros((rxn.size, 3), dtype=np.float16) print ('Saving to: {}'.format(savefn)) h5file = h5py.File(savefn, 'w') h5file.create_dataset('obstimes', data=obstimes_unix) h5file.create_dataset('stec', data=TEC, compression='gzip', compression_opts=9) h5file.create_dataset('res', data=DTEC, compression='gzip', compression_opts=9) h5file.create_dataset('roti', data=ROTI, compression='gzip', compression_opts=9) h5file.create_dataset('az', data=AZ, compression='gzip', compression_opts=9) h5file.create_dataset('el', data=EL, compression='gzip', compression_opts=9) h5file.create_dataset('rx_positions', data=RXP, compression='gzip', compression_opts=9) asciiListN = [n.encode("ascii", "ignore") for n in rxn] h5file.create_dataset('rx_name', (len(asciiListN),1),'S4', asciiListN) h5file.close() del TEC, DTEC, ROTI, AZ, EL, RXP print ('Arrays erased') for irx, rx in enumerate(rxn): if irx == 0: with open(logfn, 'a') as LOG: LOG.write('Processing {}/{}.\n'.format(irx+1, rxn.size)) LOG.close() else: with open(logfn, 'a') as LOG: LOG.write('Processing {}/{}. It took {} to process last RX data. \n'.format(irx+1, rxn.size, datetime.now()-t0)) LOG.close() t0 = datetime.now() try: D = h5py.File(F, 'r') idrx = np.isin(rxn_all, rx) sv_all = np.asanyarray([row[13] for row in D['Data/Table Layout'][idrx]]) svn = np.unique(sv_all) rx_lat = D['Data/Table Layout'][idrx][0][-4] rx_lon = D['Data/Table Layout'][idrx][0][-3] D.close() h5file = h5py.File(savefn, 'a') h5file['rx_positions'][irx, 0] = rx_lat h5file['rx_positions'][irx, 1] = rx_lon h5file['rx_positions'][irx, 2] = 0 h5file.close() del rx_lat, rx_lon for isv, sv in enumerate(svn): vtec = np.nan * np.ones(obstimes.size, dtype=np.float16) elv = np.nan * np.ones(obstimes.size, dtype=np.float16) azm = np.nan * np.ones(obstimes.size, dtype=np.float16) idsv = np.isin(sv_all, sv) ids = sv_list[str(sv)] D = h5py.File(F, 'r') t = np.asanyarray([datetime.utcfromtimestamp(row[9]) for row in D['Data/Table Layout'][idrx][idsv] ]) idt = np.isin(obstimes, t) vtec[idt] = np.asanyarray([row[18] for row in D['Data/Table Layout'][idrx][idsv] ]) elv[idt] = np.asanyarray([row[-5] for row in D['Data/Table Layout'][idrx][idsv] ]) azm[idt] = np.asanyarray([row[-6] for row in D['Data/Table Layout'][idrx][idsv] ]) D.close() idel0 = np.nan_to_num(elv) < el_mask_in idel = np.nan_to_num(elv) < el_mask vtec[idel0] = np.nan try: idx, intervals = getIntervals(vtec, maxgap=5, maxjump=maxjump) tecd, err_list = tecdPerLOS(vtec, intervals, polynom_list=polynom_list, eps=eps) tecd[idel] = np.nan rot = np.hstack((np.nan, (np.diff(vtec) / tsps))) roti = scintillation.sigmaTEC(rot, 10) # 5 min roti[idel] = np.nan vtec[idel] = np.nan h5file = h5py.File(savefn, 'a') h5file['stec'][:, isv, irx] = vtec h5file['roti'][:, isv, irx] = roti h5file['res'][:, isv, irx] = tecd h5file['el'][:, isv, irx] = elv h5file['az'][:, isv, irx] = azm h5file.close() del vtec, tecd, elv, azm, roti, rot, idx, intervals, idel0, idel except: del vtec, elv, azm del idrx, idsv, ids except: pass with open(logfn, 'a') as LOG: LOG.write('Processing Done') LOG.close() return 0
def returnGlobalTEC(date='', datafolder='', timelim=[]): if isinstance(date, str) and date != '': try: yy = date[2:4] mm = date[5:7] dd = date[-2:] except: raise ('Date has to be given in a format YYYY-MM-DD') if datafolder == '' or datafolder is None: datafolder = 'C:\\Users\\smrak\\Google Drive\\BU\\Projects\\steve\\data\\gps\\' # Open the data try: if os.path.isdir(datafolder): fnstruct = 'gps' + yy + mm + dd + 'g.002.hdf5' fn = datafolder + fnstruct elif os.path.isfile(datafolder): fn = datafolder else: raise ('Something went wrong. datafolder format is not recognized') f = h5py.File(fn, 'r') obstimes = f['Data/Table Layout']['ut2_unix'] xgrid = arange(-180, 180) ygrid = arange(-90, 90) except Exception as e: raise (e) # Time resolution if timelim is None or len(timelim) == 0: iterate = obstimes elif len(timelim) > 0 and isinstance(timelim[0], datetime): tlim = gu.datetime2posix(timelim) IDT = (obstimes >= tlim[0]) & (obstimes <= tlim[1]) iterate = unique(obstimes[IDT]) else: raise ('timelim argument has to be a dateime.datetime object, with \ with a length 2 (start,stop)') # Make an empty array to read in the data imstack = nan * ones((iterate.shape[0], xgrid.shape[0], ygrid.shape[0])) for i, t in enumerate(iterate): print('Reading in image {}/{}'.format(i + 1, iterate.shape[0])) im = nan * ones((xgrid.shape[0], ygrid.shape[0])) data_t = f['Data/Table Layout']['ut2_unix'] idt = abs(data_t - t).argmin() idT = isin(data_t, data_t[idt]) data_lon = f['Data/Table Layout']['glon'][idT] data_lat = f['Data/Table Layout']['gdlat'][idT] data_tec = f['Data/Table Layout']['tec'][idT] for k in range(data_tec.shape[0]): idx = where(xgrid == data_lon[k]) idy = where(ygrid == data_lat[k]) im[idx, idy] = data_tec[k] imstack[i, :, :] = im dt = array([datetime.utcfromtimestamp(t) for t in iterate]) out = {'time': dt, 'xgrid': xgrid, 'ygrid': ygrid, 'tecim': imstack} return out
tidim = plt.pcolormesh(xgrid, ygrid, IM.T, cmap=cmap, transform=ccrs.PlateCarree()) plt.clim(clim) # Fig utils cbar = plt.colorbar( mappable=tidim, #cax=cax, ticks=[clim[0], clim[0] / 2, 0, clim[1] / 2, clim[1]]) cbar.set_label('$\Delta$TEC [TECu]') if MASK: from pyGnss import gnssUtils # dt = parser.parse(t[i]) tposix = gnssUtils.datetime2posix([t[i]]) mask_x, mask_y, mask = gps2d.getEUVMask( tposix[0], EUVDIR= 'C:\\Users\\smrak\\Google Drive\BU\\Projects\\Eclipse2017\\data\\Drob\\HiResFull300\\' ) if isinstance(mask, ndarray): penumbra_levels = [0.2, 1, 50] levels1 = linspace(penumbra_levels[0], penumbra_levels[1], penumbra_levels[2]) lw = 1 plt.contour( mask_x, mask_y, mask.T, levels1,
def createTimeArray(timelim,Ts=1): ts = gnssUtils.datetime2posix(timelim) t = range(int(ts[0]), int(ts[1])+1) t = np.arange(int(ts[0]), int(ts[1])+1, Ts) return t
rxlistfn = stream.get('rxlist') savefn = stream.get('savefn') DATADIR = stream.get('datadir') + year + '/' + day + '/' NAVDIR = stream.get('navdir') if (savefn[-3:] != '.h5') or (savefn[-5:] != '.hdf5'): savefn += '.h5' rxlist, rxpos = ec.getRxListCoordinates(filename=rxlistfn) rxlist, rxpos = ec.rxFilter(rxlist, rxpos, latlim=latlim, lonlim=lonlim) print ("Number of receivers in the list: ", len(rxlist)) sv = np.arange(1,33) tlim = [datetime.datetime.strptime('{} {} 0 0 0'.format(year, day),'%Y %j %H %M %S'), datetime.datetime.strptime('{} {} 0 0 0'.format(year, str(int(day)+1)),'%Y %j %H %M %S')] obstimes_ts = gnssUtils.datetime2posix(tlim) time_save = np.arange(obstimes_ts[0],obstimes_ts[1],Ts) residuals = np.nan*np.zeros((time_save.shape[0], sv.shape[0], len(rxlist))) latitudes = np.nan*np.zeros((time_save.shape[0], sv.shape[0], len(rxlist))) longitudes = np.nan*np.zeros((time_save.shape[0], sv.shape[0], len(rxlist))) for j in range(len(rxlist)): print ('Processing station: '+str(j+1)+'/'+str(len(rxlist))) try: rx = rxlist[j] yamlfile = DATADIR+'/'+ rx + '.yaml' navfile = NAVDIR + '/brdc' + day + '0.'+year[-2:]+'n' hdffile = DATADIR + rx + '.h5' data = read_hdf(hdffile)