def __init__(self,flist=[],vlist=[],grid=Grid(), time=sp.array(0,dtype='|O'),\ undef=1e15, name='',mode='r'): print 'opening GADset' super(GADset, self).__init__(flist, vlist, grid, time, undef, name) # File map self.fmap = sp.zeros((time.size, 2), dtype=sp.int32) self.files = sp.empty(len(flist), dtype='|O') srec = 0 rectot = 0 for i, fname in enumerate(flist): try: self.files[i] = sp.memmap(fname, dtype=vlist, mode=mode) nrec = self.files[i].shape[0] except IOError, err: print fname print err nrec = 1 erec = srec + nrec self.fmap[srec:erec, 0] = i self.fmap[srec:erec, 1] = sp.arange(srec, erec) - rectot srec = erec rectot += nrec
def __init__(self): name='Reynolds MERRA' undef=-999.0 flist=['/discover/nobackup/projects/gmao/share/dao_ops/fvInput/g5gcm/bcs/realtime/SST/360x180/dataoceanfile_MERRA_sst_1971-current.360x180.LE'] lon=sp.arange(-179.5,180,1); lat=sp.arange(-89.5,90,1); lev=sp.zeros(1) grid=dset.Grid(lon,lat,lev) vlist=[('','i4',1), ('head','f4',14), ('','i4',1), ('','i4',1), ('sst','f4',grid.dims), ('','i4',1)] time=sp.repeat(datetime.date(1,1,1),1732) a=sp.memmap(flist[0],dtype=vlist,mode='r') for i,hh in enumerate(a['head']): time[i]=datetime.date(hh[0],hh[1],hh[2]) a.close() super(Ctl,self).__init__(flist,vlist,grid,time,undef,name)
def plot_saved(): import matplotlib import matplotlib.pyplot as plt data = sp.memmap(get_filename(), dtype=sp.float64, mode='r', shape=(N_samp + 2, N_steps / res_every + 1, N)) exa = data[-1] data = data[:-2] fins = sp.array([d[plot_res] for d in data if not sp.all(d[plot_res] == 0)]) nsam = len(fins) print("Samples:", nsam) av = fins.sum(axis=0) / nsam av_var = 1./(nsam - 1) / nsam * sp.sum((fins/nsam - av)**2, axis=0) av_e1 = av + sp.sqrt(av_var) av_e2 = av - sp.sqrt(av_var) plt.figure() pav = plt.plot(av, 'k-')[0] plt.plot(av_e1, 'k--') plt.plot(av_e2, 'k--') if not sp.all(exa[-1] == 0): pexa = plt.plot(exa[-1], 'r-')[0] plt.legend([pexa, pav], ["Density matrix", "Sample average"]) plt.ylim((-1, 1)) plt.xlabel(r"$n$") plt.ylabel(r"$\langle \sigma^z_n \rangle$") plt.show()
def _allocate_samples( self, name, shape ): """Allocate for (shape) samples""" # Save samples in a temporary mem-mapped array, fname save in # the metadata "params" if self.dname is None: self.dname = tempfile.mkdtemp() arr = sc.memmap( os.path.join(self.dname,name), mode="w+", shape=shape, dtype = sc.double ) return arr
def fromfile(self,varname,iind=slice(None),\ jind=slice(None),kind=slice(None),tind=slice(None)): ii, jj, kk, tt = utl.scalar2slice(iind, jind, kind, tind) var = self.subset(ii, jj, kk, tt) var.name = varname a = sp.memmap(self.flist[0], dtype=self.vlist) var.data = sp.ma.masked_values( a[tt][varname][:, kk][:, :, jj][:, :, :, ii], self.undef) return var
def _allocate_samples(self, name, shape): """Allocate for (shape) samples""" # Save samples in a temporary mem-mapped array, fname save in # the metadata "params" if self.dname is None: self.dname = tempfile.mkdtemp() arr = sc.memmap(os.path.join(self.dname, name), mode="w+", shape=shape, dtype=sc.double) return arr
def writer(q): df = sp.memmap(get_filename(), dtype=sp.float64, mode='w+', shape=(N_samp + 2, N_steps / res_every + 1, N)) while True: data = q.get() if data is None: break num = data[0] i = data[1] ys = data[2] df[num, i / res_every, :] = ys if i == N_steps: df.flush() print("Sample", num, "finished. Data saved.") del df
print('#' * 80) print('# Statistics and profile of pan-genome:') print( '# The methods can be found in Hu X, et al. Trajectory and genomic determinants of fungal-pathogen speciation and host adaptation.' ) print('#') print('# statistic of core, shared and specific genes:') print('\t'.join(['# Feature', 'core', 'shared', 'specific', 'taxon'])) print('\t'.join(map(str, ['# Number', core, shar, spec, N]))) # print flag, N #_o.close() # calculate the core, share and specific gene's profile #print('flag and N', flag, N) fp = np.memmap('pan.npy', mode='r+', shape=(flag, N), dtype='int32') # print 'fp is', fp[:] mat = np.asarray(fp, dtype='bool') mat = np.asarray(mat, dtype='int8') #mat[mat>0] = 1 #mat = np.asarray([elem[2:] for elem in outputs]) # print ts, tc, Ts, Tc # print mat def pan_feature0(x, ts=.05, tc=.95): n, d = x.shape idx = list(range(d)) index = []
# Load Wavelengths wavelengths = np.loadtxt( 'C:\\Users\\MonicaVermillion\\Documents\\JPL\\prism\\GrizzlyBay_Batch\\data\\wavelength.txt' ) wave = wavelengths[:, 1] # Load in the lines and samples to grab from the image overlap = np.loadtxt('lines-samples-950.txt') lines = overlap[:, 0] # Rows samples = overlap[:, 1] # Columns # Load the data and convert to Band-In-Pixel interleave filename = 'C:\\Users\\MonicaVermillion\\Documents\\JPL\\prism\\prm\\prm20140428t230950_rdn_v1c\\prm20140428t230950_rdn_v1c_img' rows, bands, cols = 4944, 246, 721 mm = s.memmap(filename, dtype=s.float32, mode='r', shape=(rows, bands, cols)) d = s.asarray(mm, dtype=s.float32).copy() # Takes a little while to run # Grab the pixels that overlap: #for j,k in enumerate(lines) #col = samples[j] #pixel = d[samp,k,4:-1] t_line = int(lines[0]) # Test line t_sample = int(samples[0]) # Test sample pixel = d[t_line, 4:, t_sample] #plt.plot(pixel) rad_stack = np.column_stack((wave, pixel)) #for h in range(3):
time.sleep(10) print('pulling file') call([ 'scp', 'mdsplus@raspicam4:/home/mdsplus/out' + str(shot) + '.dat', loc + 'out.dat' ]) call(['chmod', '777', loc + 'out.dat']) print('removing file from raspicam') call([ 'ssh', 'mdsplus@raspicam4', 'rm', '/home/mdsplus/out' + str(shot) + '.dat' ]) # load and unpack .dat file data = scipy.memmap(loc + 'out.dat', dtype='uint8', shape=(frames, res[1], res[0])) #data2 = scipy.transpose(data,(1,2,0)) #reorder data for use in mdsplus #this call makes me sad, because its in a great form to begin with... timebase = scipy.linspace(0, frames / framerate, frames) + start # upload to tree using putdata dummy = MDSplus.Data.compile('build_signal($,*,$)', data, timebase) tree.getNode(node + ':frames').putData(dummy) MDSplus.Event.seteven(complete) # cleanup print('cleaning temporary files') call(['rm', loc + 'out.dat'])
#jm,im=loni.shape loni, lati = sp.meshgrid(sp.arange(-179.875, 180, 0.25), sp.arange(-89.875, 90, 0.25)) loni[loni > 79.875] -= 360. jm, im = 720, 1440 vlist = [] vlist.append(('head', 'f4', 16)) vlist.append(('h1', 'i4', 1)) vlist.append(('kpar', 'f4', (jm, im))) vlist.append(('h2', 'i4', 1)) #a=sp.memmap('/discover/nobackup/yvikhlia/coupled/Forcings/a288x181_o720x410/SEAWIFS_KPAR_mon_clim.720x410',dtype=vlist,mode='r') a = sp.memmap( '/discover/nobackup/projects/gmao/share/dao_ops/fvInput/g5gcm/bcs/realtime/SST/1440x720/SEAWIFS_KPAR_mon_clim.1440x720', dtype=vlist, mode='r') go = nc.Dataset( '/home/yvikhlya/nobackup/coupled/4_0_beta10/GEOSagcm/src/GEOSgcs_GridComp/GEOSgcm_GridComp/GEOSagcm_GridComp/GEOSphysics_GridComp/GEOSsurface_GridComp/Shared/Raster/data/MOM/1440x1080/grid_spec.nc' ) lono = go.variables['x_T'][:] lato = go.variables['y_T'][:] jm, im = lono.shape vlist = [] vlist.append(('head', 'f4', 16)) vlist.append(('h1', 'i4', 1)) vlist.append(('kpar', 'f4', (jm, im))) vlist.append(('h2', 'i4', 1)) b = sp.zeros(14, dtype=vlist)
otype = [] otype.append(('head', 'f4', 16)) otype.append(('h1', 'i4', 1)) otype.append(('var', 'f4', (jmo, imo))) otype.append(('h2', 'i4', 1)) odir = os.environ['NOBACKUP'] + '/workdir/SST' try: os.makedirs(odir) except OSError: pass for ff in iflist: of = ff.replace(str(imi) + 'x' + str(jmi), str(imo) + 'x' + str(jmo)) print ff + ' --> ' + of aa = sp.memmap(idir + '/' + ff, dtype=itype, mode='r') zout = sp.zeros(aa.size, dtype=otype) zout['h1'] = zout['h2'] = imo * jmo * 4 zout['head'] = aa['head'] zout['head'][:, 13] = imo zout['head'][:, 14] = jmo for ii, zin in enumerate(aa['var']): zout['var'][ii] = sp.where( wet == 0, 1e15, interpolate.griddata(zip(loni[mask], lati[mask]), zin[mask], (lono, lato), method='nearest', fill_value=1e15)) zout.tofile(odir + '/' + of)
def extractions(inputfile, labels, output, chunksize, flag): """...""" in_file = inputfile lbl_file = labels out_file = output nchunk = chunksize dtm = {'4': s.float32, '5': s.float64} # Open input data, get dimensions in_img = envi.open(in_file + '.hdr', in_file) meta = in_img.metadata nl, nb, ns = [int(meta[n]) for n in ('lines', 'bands', 'samples')] img_mm = in_img.open_memmap(interleave='source', writable=False) lbl_img = envi.open(lbl_file + '.hdr', lbl_file) labels = lbl_img.read_band(0) nout = len(s.unique(labels)) # reindex from zero to n #lbl = s.sort(s.unique(labels.flat)) #idx = s.arange(len(lbl)) #nout = len(lbl) # for i, L in enumerate(lbl): # labels[labels==L] = i # Iterate through image "chunks," segmenting as we go next_label = 1 extracted = s.zeros(nout) > 1 out = s.zeros((nout, nb)) counts = s.zeros((nout)) for lstart in s.arange(0, nl, nchunk): del img_mm img_mm = in_img.open_memmap(interleave='source', writable=False) # Which labels will we extract? ignore zero index lend = min(lstart + nchunk, nl) active = s.unique(labels[lstart:lend, :]) active = active[active >= 1] # Handle labels extending outside our chunk by expanding margins active_area = s.zeros(labels.shape) lstart_adjust, lend_adjust = lstart, lend for i in active: active_area[labels == i] = True active_locs = s.where(active_area) lstart_adjust = min(active_locs[0]) lend_adjust = max(active_locs[0]) + 1 chunk_inp = s.array(img_mm[lstart_adjust:lend_adjust, :, :]) if meta['interleave'] == 'bil': chunk_inp = chunk_inp.transpose((0, 2, 1)) chunk_lbl = s.array(labels[lstart_adjust:lend_adjust, :]) for i in active: idx = int(i) out[idx, :] = 0 locs = s.where(chunk_lbl == i) for row, col in zip(locs[0], locs[1]): out[idx, :] = out[idx, :] + s.squeeze(chunk_inp[row, col, :]) counts[idx] = len(locs[0]) out = s.array((out.T / counts[s.newaxis, :]).T, dtype=s.float32) out[s.logical_not(s.isfinite(out))] = flag meta["lines"] = str(nout) meta["bands"] = str(nb) meta["samples"] = '1' meta["interleave"] = "bil" out_img = envi.create_image(out_file + '.hdr', metadata=meta, ext='', force=True) out_mm = s.memmap(out_file, dtype=dtm[meta['data type']], mode='w+', shape=(nout, 1, nb)) if dtm[meta['data type']] == s.float32: out_mm[:, 0, :] = s.array(out, s.float32) else: out_mm[:, 0, :] = s.array(out, s.float64)