def __init__(self,flist=[],vlist=[],grid=Grid(), time=sp.array(0,dtype='|O'),\
undef=1e15, name='',mode='r'):
print 'opening GADset'
super(GADset, self).__init__(flist, vlist, grid, time, undef, name)
# File map
self.fmap = sp.zeros((time.size, 2), dtype=sp.int32)
self.files = sp.empty(len(flist), dtype='|O')
srec = 0
rectot = 0
for i, fname in enumerate(flist):
try:
self.files[i] = sp.memmap(fname, dtype=vlist, mode=mode)
nrec = self.files[i].shape[0]
except IOError, err:
print fname
print err
nrec = 1
erec = srec + nrec
self.fmap[srec:erec, 0] = i
self.fmap[srec:erec, 1] = sp.arange(srec, erec) - rectot
srec = erec
rectot += nrec
def __init__(self):
name='Reynolds MERRA'
undef=-999.0
flist=['/discover/nobackup/projects/gmao/share/dao_ops/fvInput/g5gcm/bcs/realtime/SST/360x180/dataoceanfile_MERRA_sst_1971-current.360x180.LE']
lon=sp.arange(-179.5,180,1); lat=sp.arange(-89.5,90,1); lev=sp.zeros(1)
grid=dset.Grid(lon,lat,lev)
vlist=[('','i4',1),
('head','f4',14),
('','i4',1),
('','i4',1),
('sst','f4',grid.dims),
('','i4',1)]
time=sp.repeat(datetime.date(1,1,1),1732)
a=sp.memmap(flist[0],dtype=vlist,mode='r')
for i,hh in enumerate(a['head']):
time[i]=datetime.date(hh[0],hh[1],hh[2])
a.close()
super(Ctl,self).__init__(flist,vlist,grid,time,undef,name)
def plot_saved():
import matplotlib
import matplotlib.pyplot as plt
data = sp.memmap(get_filename(), dtype=sp.float64, mode='r', shape=(N_samp + 2, N_steps / res_every + 1, N))
exa = data[-1]
data = data[:-2]
fins = sp.array([d[plot_res] for d in data if not sp.all(d[plot_res] == 0)])
nsam = len(fins)
print("Samples:", nsam)
av = fins.sum(axis=0) / nsam
av_var = 1./(nsam - 1) / nsam * sp.sum((fins/nsam - av)**2, axis=0)
av_e1 = av + sp.sqrt(av_var)
av_e2 = av - sp.sqrt(av_var)
plt.figure()
pav = plt.plot(av, 'k-')[0]
plt.plot(av_e1, 'k--')
plt.plot(av_e2, 'k--')
if not sp.all(exa[-1] == 0):
pexa = plt.plot(exa[-1], 'r-')[0]
plt.legend([pexa, pav], ["Density matrix", "Sample average"])
plt.ylim((-1, 1))
plt.xlabel(r"$n$")
plt.ylabel(r"$\langle \sigma^z_n \rangle$")
plt.show()
def _allocate_samples( self, name, shape ):
"""Allocate for (shape) samples"""
# Save samples in a temporary mem-mapped array, fname save in
# the metadata "params"
if self.dname is None:
self.dname = tempfile.mkdtemp()
arr = sc.memmap( os.path.join(self.dname,name), mode="w+", shape=shape, dtype = sc.double )
return arr
def fromfile(self,varname,iind=slice(None),\
jind=slice(None),kind=slice(None),tind=slice(None)):
ii, jj, kk, tt = utl.scalar2slice(iind, jind, kind, tind)
var = self.subset(ii, jj, kk, tt)
var.name = varname
a = sp.memmap(self.flist[0], dtype=self.vlist)
var.data = sp.ma.masked_values(
a[tt][varname][:, kk][:, :, jj][:, :, :, ii], self.undef)
return var
def _allocate_samples(self, name, shape):
"""Allocate for (shape) samples"""
# Save samples in a temporary mem-mapped array, fname save in
# the metadata "params"
if self.dname is None:
self.dname = tempfile.mkdtemp()
arr = sc.memmap(os.path.join(self.dname, name),
mode="w+",
shape=shape,
dtype=sc.double)
return arr
def writer(q):
df = sp.memmap(get_filename(), dtype=sp.float64, mode='w+', shape=(N_samp + 2, N_steps / res_every + 1, N))
while True:
data = q.get()
if data is None:
break
num = data[0]
i = data[1]
ys = data[2]
df[num, i / res_every, :] = ys
if i == N_steps:
df.flush()
print("Sample", num, "finished. Data saved.")
del df
print('#' * 80)
print('# Statistics and profile of pan-genome:')
print(
'# The methods can be found in Hu X, et al. Trajectory and genomic determinants of fungal-pathogen speciation and host adaptation.'
)
print('#')
print('# statistic of core, shared and specific genes:')
print('\t'.join(['# Feature', 'core', 'shared', 'specific', 'taxon']))
print('\t'.join(map(str, ['# Number', core, shar, spec, N])))
# print flag, N
#_o.close()
# calculate the core, share and specific gene's profile
#print('flag and N', flag, N)
fp = np.memmap('pan.npy', mode='r+', shape=(flag, N), dtype='int32')
# print 'fp is', fp[:]
mat = np.asarray(fp, dtype='bool')
mat = np.asarray(mat, dtype='int8')
#mat[mat>0] = 1
#mat = np.asarray([elem[2:] for elem in outputs])
# print ts, tc, Ts, Tc
# print mat
def pan_feature0(x, ts=.05, tc=.95):
n, d = x.shape
idx = list(range(d))
index = []
# Load Wavelengths
wavelengths = np.loadtxt(
'C:\\Users\\MonicaVermillion\\Documents\\JPL\\prism\\GrizzlyBay_Batch\\data\\wavelength.txt'
)
wave = wavelengths[:, 1]
# Load in the lines and samples to grab from the image
overlap = np.loadtxt('lines-samples-950.txt')
lines = overlap[:, 0] # Rows
samples = overlap[:, 1] # Columns
# Load the data and convert to Band-In-Pixel interleave
filename = 'C:\\Users\\MonicaVermillion\\Documents\\JPL\\prism\\prm\\prm20140428t230950_rdn_v1c\\prm20140428t230950_rdn_v1c_img'
rows, bands, cols = 4944, 246, 721
mm = s.memmap(filename, dtype=s.float32, mode='r', shape=(rows, bands, cols))
d = s.asarray(mm, dtype=s.float32).copy() # Takes a little while to run
# Grab the pixels that overlap:
#for j,k in enumerate(lines)
#col = samples[j]
#pixel = d[samp,k,4:-1]
t_line = int(lines[0]) # Test line
t_sample = int(samples[0]) # Test sample
pixel = d[t_line, 4:, t_sample]
#plt.plot(pixel)
rad_stack = np.column_stack((wave, pixel))
#for h in range(3):
time.sleep(10)
print('pulling file')
call([
'scp', 'mdsplus@raspicam4:/home/mdsplus/out' + str(shot) + '.dat',
loc + 'out.dat'
])
call(['chmod', '777', loc + 'out.dat'])
print('removing file from raspicam')
call([
'ssh', 'mdsplus@raspicam4', 'rm', '/home/mdsplus/out' + str(shot) + '.dat'
])
# load and unpack .dat file
data = scipy.memmap(loc + 'out.dat',
dtype='uint8',
shape=(frames, res[1], res[0]))
#data2 = scipy.transpose(data,(1,2,0)) #reorder data for use in mdsplus
#this call makes me sad, because its in a great form to begin with...
timebase = scipy.linspace(0, frames / framerate, frames) + start
# upload to tree using putdata
dummy = MDSplus.Data.compile('build_signal($,*,$)', data, timebase)
tree.getNode(node + ':frames').putData(dummy)
MDSplus.Event.seteven(complete)
# cleanup
print('cleaning temporary files')
call(['rm', loc + 'out.dat'])
#jm,im=loni.shape
loni, lati = sp.meshgrid(sp.arange(-179.875, 180, 0.25),
sp.arange(-89.875, 90, 0.25))
loni[loni > 79.875] -= 360.
jm, im = 720, 1440
vlist = []
vlist.append(('head', 'f4', 16))
vlist.append(('h1', 'i4', 1))
vlist.append(('kpar', 'f4', (jm, im)))
vlist.append(('h2', 'i4', 1))
#a=sp.memmap('/discover/nobackup/yvikhlia/coupled/Forcings/a288x181_o720x410/SEAWIFS_KPAR_mon_clim.720x410',dtype=vlist,mode='r')
a = sp.memmap(
'/discover/nobackup/projects/gmao/share/dao_ops/fvInput/g5gcm/bcs/realtime/SST/1440x720/SEAWIFS_KPAR_mon_clim.1440x720',
dtype=vlist,
mode='r')
go = nc.Dataset(
'/home/yvikhlya/nobackup/coupled/4_0_beta10/GEOSagcm/src/GEOSgcs_GridComp/GEOSgcm_GridComp/GEOSagcm_GridComp/GEOSphysics_GridComp/GEOSsurface_GridComp/Shared/Raster/data/MOM/1440x1080/grid_spec.nc'
)
lono = go.variables['x_T'][:]
lato = go.variables['y_T'][:]
jm, im = lono.shape
vlist = []
vlist.append(('head', 'f4', 16))
vlist.append(('h1', 'i4', 1))
vlist.append(('kpar', 'f4', (jm, im)))
vlist.append(('h2', 'i4', 1))
b = sp.zeros(14, dtype=vlist)
otype = []
otype.append(('head', 'f4', 16))
otype.append(('h1', 'i4', 1))
otype.append(('var', 'f4', (jmo, imo)))
otype.append(('h2', 'i4', 1))
odir = os.environ['NOBACKUP'] + '/workdir/SST'
try:
os.makedirs(odir)
except OSError:
pass
for ff in iflist:
of = ff.replace(str(imi) + 'x' + str(jmi), str(imo) + 'x' + str(jmo))
print ff + ' --> ' + of
aa = sp.memmap(idir + '/' + ff, dtype=itype, mode='r')
zout = sp.zeros(aa.size, dtype=otype)
zout['h1'] = zout['h2'] = imo * jmo * 4
zout['head'] = aa['head']
zout['head'][:, 13] = imo
zout['head'][:, 14] = jmo
for ii, zin in enumerate(aa['var']):
zout['var'][ii] = sp.where(
wet == 0, 1e15,
interpolate.griddata(zip(loni[mask], lati[mask]),
zin[mask], (lono, lato),
method='nearest',
fill_value=1e15))
zout.tofile(odir + '/' + of)
def extractions(inputfile, labels, output, chunksize, flag):
"""..."""
in_file = inputfile
lbl_file = labels
out_file = output
nchunk = chunksize
dtm = {'4': s.float32, '5': s.float64}
# Open input data, get dimensions
in_img = envi.open(in_file + '.hdr', in_file)
meta = in_img.metadata
nl, nb, ns = [int(meta[n]) for n in ('lines', 'bands', 'samples')]
img_mm = in_img.open_memmap(interleave='source', writable=False)
lbl_img = envi.open(lbl_file + '.hdr', lbl_file)
labels = lbl_img.read_band(0)
nout = len(s.unique(labels))
# reindex from zero to n
#lbl = s.sort(s.unique(labels.flat))
#idx = s.arange(len(lbl))
#nout = len(lbl)
# for i, L in enumerate(lbl):
# labels[labels==L] = i
# Iterate through image "chunks," segmenting as we go
next_label = 1
extracted = s.zeros(nout) > 1
out = s.zeros((nout, nb))
counts = s.zeros((nout))
for lstart in s.arange(0, nl, nchunk):
del img_mm
img_mm = in_img.open_memmap(interleave='source', writable=False)
# Which labels will we extract? ignore zero index
lend = min(lstart + nchunk, nl)
active = s.unique(labels[lstart:lend, :])
active = active[active >= 1]
# Handle labels extending outside our chunk by expanding margins
active_area = s.zeros(labels.shape)
lstart_adjust, lend_adjust = lstart, lend
for i in active:
active_area[labels == i] = True
active_locs = s.where(active_area)
lstart_adjust = min(active_locs[0])
lend_adjust = max(active_locs[0]) + 1
chunk_inp = s.array(img_mm[lstart_adjust:lend_adjust, :, :])
if meta['interleave'] == 'bil':
chunk_inp = chunk_inp.transpose((0, 2, 1))
chunk_lbl = s.array(labels[lstart_adjust:lend_adjust, :])
for i in active:
idx = int(i)
out[idx, :] = 0
locs = s.where(chunk_lbl == i)
for row, col in zip(locs[0], locs[1]):
out[idx, :] = out[idx, :] + s.squeeze(chunk_inp[row, col, :])
counts[idx] = len(locs[0])
out = s.array((out.T / counts[s.newaxis, :]).T, dtype=s.float32)
out[s.logical_not(s.isfinite(out))] = flag
meta["lines"] = str(nout)
meta["bands"] = str(nb)
meta["samples"] = '1'
meta["interleave"] = "bil"
out_img = envi.create_image(out_file + '.hdr',
metadata=meta,
ext='',
force=True)
out_mm = s.memmap(out_file,
dtype=dtm[meta['data type']],
mode='w+',
shape=(nout, 1, nb))
if dtm[meta['data type']] == s.float32:
out_mm[:, 0, :] = s.array(out, s.float32)
else:
out_mm[:, 0, :] = s.array(out, s.float64)
