def assertDataAlmostEqual(self, data, reference_filename, **kwargs):
reference_path = self.get_result_path(reference_filename)
if self._check_reference_file(reference_path):
kwargs.setdefault('err_msg', 'Reference file %s' % reference_path)
with open(reference_path, 'r') as reference_file:
stats = json.load(reference_file)
self.assertEqual(stats.get('shape', []), list(data.shape))
self.assertEqual(stats.get('masked', False),
ma.is_masked(data))
nstats = np.array((stats.get('mean', 0.), stats.get('std', 0.),
stats.get('max', 0.), stats.get('min', 0.)),
dtype=np.float_)
if math.isnan(stats.get('mean', 0.)):
self.assertTrue(math.isnan(data.mean()))
else:
data_stats = np.array((data.mean(), data.std(),
data.max(), data.min()),
dtype=np.float_)
self.assertArrayAllClose(nstats, data_stats, **kwargs)
else:
self._ensure_folder(reference_path)
stats = collections.OrderedDict([
('std', np.float_(data.std())),
('min', np.float_(data.min())),
('max', np.float_(data.max())),
('shape', data.shape),
('masked', ma.is_masked(data)),
('mean', np.float_(data.mean()))])
with open(reference_path, 'w') as reference_file:
reference_file.write(json.dumps(stats))
def SmartGen(device_id, k,device_user,generate_device_user_likelihood): user_candidate = [] precision = [] candidates_n = estimation = candidates_l = 0.0 users = device_user.get(device_id) likelihood =estimation_list = [] for user in users: likelihood.append(generate_device_user_likelihood.get(user)) for k in range(1, len(users)): for i in ~np.argsort(likelihood)[:k]: user = users[i] user_candidate.append(user) candidates_n += len(AllCookie.get(user)) candidates_l += generate_device_user_likelihood.get(user) for i in ~np.argsort(likelihood)[:k]: user = users[i] precision = np.float_(len(AllCookie.get(user))) / np.float_(candidates_n) estimation += (generate_device_user_likelihood.get(user)*1.0 / candidates_l)* ((1.25*precision) / (0.25*precision + 1.0)) estimation_list.append(estimation) k_final = ~np.argsort(estimation_list) cookie_final = [] for i in ~np.argsort(likelihood)[:k_final]: user = users[i] cookie_candidates = HandleCookie.get(user) for cookie in cookie_candidates: cookie_final.append(cookie) return cookie_final
def matrix_mul(X1, X2, shard_size=5000):
""" Calculate matrix multiplication for big matrix,
X1 and X2 are sliced into pieces with shard_size rows(columns)
then multiplied together and concatenated to the proper size
"""
X1 = np.float_(X1)
X2 = np.float_(X2)
X1_shape = X1.shape
X2_shape = X2.shape
assert X1_shape[1] == X2_shape[0]
X1_iter = X1_shape[0] // shard_size + 1
X2_iter = X2_shape[1] // shard_size + 1
all_result = np.zeros((1,))
for X1_id in range(X1_iter):
result = np.zeros((1,))
for X2_id in range(X2_iter):
partial_result = np.matmul(
X1[X1_id * shard_size:min((X1_id + 1) *
shard_size, X1_shape[0]), :],
X2[:, X2_id * shard_size:min((X2_id + 1) *
shard_size, X2_shape[1])])
# calculate matrix multiplicatin on slices
if result.size == 1:
result = partial_result
else:
result = np.concatenate((result, partial_result), axis=1)
# concatenate the slices together
del partial_result
if all_result.size == 1:
all_result = result
else:
all_result = np.concatenate((all_result, result), axis=0)
del result
return all_result
def __init__(self, eps_par=numpy.float_(0.),
mask_var=numpy.float_(1),xlabel=None,ylabel=None):
super(MyEstimator, self).__init__()
# self.params=dict()
# self.params['']=
# self.params['eps_par']=eps_par
# self.params['mask_var']=mask_var
#these are the parameters
self.eps_par=eps_par
self.mask_var=mask_var
self.catastrophe=None
self.dm=None
self.max_distance=None
self.mask_scale=None
self.outlier_cut=0.95
self.optimize_frac = 0.1
self.xlabel=xlabel
self.ylabel=ylabel
self.zmin = 0.6
self.zmax=1.6
self.oiimin=6e-17
def pval_KalZtest(n1,N1,n2,N2):
"""Compute p-value using Kal Z-test for count data.
Compute pval using Z-test, as published in
Kal et al, 1999, Mol Biol Cell 10:1859.
Z = (p1-p2) / sqrt( p0 * (1-p0) * (1/N1 + 1/N2) )
where p1 = n1/N1, p2=n2/N2, and p0=(n1+n2)/(N1+N2)
You reject if |Z| > Z_a/2 where a is sig lev. Here
we return the p-value itself.
"""
if n1==0 and n2==0:
return 1.0
n1 = np.float_(n1)
N1 = np.float_(N1)
n2 = np.float_(n2)
N2 = np.float_(N2)
p0 = (n1+n2)/(N1+N2)
p1 = n1/N1
p2 = n2/N2
Z = (p1-p2) / np.sqrt( p0 * (1-p0) * ((1/N1) + (1/N2)) )
pval = 2 * sp.stats.norm.cdf(-1*abs(Z))
return pval
def linearinterpolation(nstep,ndata,dt):
"""used for lienar interpolation between transportation matrices.
returns weights alpha, beta and indices of matrices.
Parameters
-------
nstep : int Number of timesteps
ndata : int Number of matrices
Returns
-------
alpha,beta : array
coefficients for interpolation
jalpha,jbeta : array
indices for interpolation
"""
t = np.zeros(nstep,dtype=np.float_)
for i in range(nstep):
t[i] = np.fmod(0 + i*dt, 1.0)
beta = np.array(nstep,dtype=np.float_)
alpha = np.array(nstep,dtype=np.float_)
w = t * ndata+0.5
beta = np.float_(np.fmod(w, 1.0))
alpha = np.float_(1.0-beta)
jalpha = np.fmod(np.floor(w)+ndata-1.0,ndata).astype(int)
jbeta = np.fmod(np.floor(w),ndata).astype(int)
return alpha,beta,jalpha,jbeta
def rotatev_aroundx(p,a): #
x=p[0]
y=p[1]
z=p[2]
cs=np.float_(np.cos(a))
cn=np.float_(np.sin(a))
return np.float_(np.array([x,cs*y-cn*z,cn*y+cs*z]))
def Init(self):
#boundary and domain condition
self.lat = io.read_PETSc_vec(self.config["-Metos3DBoundaryConditionInputDirectory"][0] + self.config["-Metos3DLatitudeFileFormat"][0])
dz = io.read_PETSc_vec(self.config["-Metos3DDomainConditionInputDirectory"][0] + self.config["-Metos3DLayerHeightFileFormat"][0])
z = io.read_PETSc_vec(self.config["-Metos3DDomainConditionInputDirectory"][0] + self.config["-Metos3DLayerDepthFileFormat"][0])
self.lsm = io.read_PETSc_mat(self.config["-Metos3DProfileInputDirectory"][0] + self.config["-Metos3DProfileMaskFile"][0])
self.fice = np.zeros((self.profiles,np.int_(self.config["-Metos3DIceCoverCount"][0])),dtype=np.float_)
for i in range(np.int_(self.config["-Metos3DIceCoverCount"][0])):
self.fice[:,i] = io.read_PETSc_vec(self.config["-Metos3DBoundaryConditionInputDirectory"][0] + (self.config["-Metos3DIceCoverFileFormat"][0] % i))
self.bc = np.zeros(2,dtype=np.float_)
self.dc = np.zeros((self.ny,2),dtype=np.float_)
self.dc[:,0] = z
self.dc[:,1] = dz
self.u = np.array(self.config["-Metos3DParameterValue"],dtype=np.float_)
self.dt = np.float_(self.config["-Metos3DTimeStep"][0])
self.nspinup = np.int_(self.config["-Metos3DSpinupCount"][0])
self.ntimestep = np.int_(self.config["-Metos3DTimeStepCount"][0])
self.matrixCount = np.int_(self.config["-Metos3DMatrixCount"][0])
self.U_PODN = np.load(self.config["-Metos3DMatrixInputDirectory"][0] +'N/'+ self.config["-Metos3DMatrixPODFileFormat"][0])
self.U_PODDOP = np.load(self.config["-Metos3DMatrixInputDirectory"][0] +'DOP/'+ self.config["-Metos3DMatrixPODFileFormat"][0])
self.U_DEIMN = np.load(self.config["-Metos3DMatrixInputDirectory"][0] +'N/'+ self.config["-Metos3DMatrixDEIMFileFormat"][0])
self.U_DEIMDOP = np.load(self.config["-Metos3DMatrixInputDirectory"][0] +'DOP/'+ self.config["-Metos3DMatrixDEIMFileFormat"][0])
self.DEIM_IndicesN = np.load(self.config["-Metos3DMatrixInputDirectory"][0] +'N/'+ self.config["-Metos3DDEIMIndicesFileFormat"][0])
self.DEIM_IndicesDOP = np.load(self.config["-Metos3DMatrixInputDirectory"][0] +'DOP/'+ self.config["-Metos3DDEIMIndicesFileFormat"][0])
self.AN = np.ndarray(shape=(self.matrixCount,self.U_PODN.shape[1],self.U_PODN.shape[1]), dtype=np.float_, order='C')
self.ADOP = np.ndarray(shape=(self.matrixCount,self.U_PODDOP.shape[1],self.U_PODDOP.shape[1]), dtype=np.float_, order='C')
for i in range(0,self.matrixCount):
self.AN[i] = np.load(self.config["-Metos3DMatrixInputDirectory"][0] +'N/'+ self.config["-Metos3DMatrixReducedFileFormat"][0] % i)
self.ADOP[i] = np.load(self.config["-Metos3DMatrixInputDirectory"][0] +'DOP/'+ self.config["-Metos3DMatrixReducedFileFormat"][0] % i)
self.PN = np.ndarray(shape=(self.matrixCount,self.U_PODN.shape[1],self.U_DEIMN.shape[1]), dtype=np.float_, order='C')
self.PDOP = np.ndarray(shape=(self.matrixCount,self.U_PODDOP.shape[1],self.U_DEIMDOP.shape[1]), dtype=np.float_, order='C')
for i in range(0,self.matrixCount):
self.PN[i] = np.load(self.config["-Metos3DMatrixInputDirectory"][0] +'N/'+ self.config["-Metos3DMatrixReducedDEINFileFormat"][0] % i)
self.PDOP[i] = np.load(self.config["-Metos3DMatrixInputDirectory"][0] +'DOP/'+ self.config["-Metos3DMatrixReducedDEINFileFormat"][0] % i)
#precomputin the interplaton indices for a year
[self.interpolation_a,self.interpolation_b,self.interpolation_j,self.interpolation_k] = util.linearinterpolation(2880,12,0.0003472222222222)
self.yN = np.ones(self.ny,dtype=np.float_) * np.float_(self.config["-Metos3DTracerInitValue"])[0]
self.yDOP = np.ones(self.ny,dtype=np.float_) * np.float_(self.config["-Metos3DTracerInitValue"])[1]
self.y_redN = np.dot(self.U_PODN.T,self.yN)
self.y_redDOP = np.dot(self.U_PODDOP.T,self.yDOP)
self.qN = np.zeros(self.DEIM_IndicesN.shape[0],dtype=np.float_)
self.qDOP = np.zeros(self.DEIM_IndicesDOP.shape[0],dtype=np.float_)
self.J,self.PJ = util.generateIndicesForNonlinearFunction(self.lsm,self.profiles,self.ny)
self.out_pathN = self.config["-Metos3DTracerOutputDirectory"][0] +self.config["-Metos3DSpinupMonitorFileFormatPrefix"][0] + self.config["-Metos3DSpinupMonitorFileFormatPrefix"][1] +self.config["-Metos3DTracerOutputFile"][0]
self.out_pathDOP = self.config["-Metos3DTracerOutputDirectory"][0] +self.config["-Metos3DSpinupMonitorFileFormatPrefix"][0] + self.config["-Metos3DSpinupMonitorFileFormatPrefix"][1] +self.config["-Metos3DTracerOutputFile"][1]
self.monitor_path = self.config["-Metos3DTracerMointorDirectory"][0] +self.config["-Metos3DSpinupMonitorFileFormatPrefix"][0] + self.config["-Metos3DSpinupMonitorFileFormatPrefix"][1] +self.config["-Metos3DTracerOutputFile"][0]
def loadDevices(trainfile,DictHandle,DictDevice,DictDevType,DictDevOs,DictCountry,DictAnnC1,DictAnnC2):
NumRows = 0
with open(trainfile,'rb') as csvfile:
spamreader=csv.reader(csvfile,delimiter=',')
spamreader.next()
for row in spamreader:
NumRows = NumRows + 1
XDevices = np.zeros((NumRows,11))
NumRows = 0
with open(trainfile,'rb') as csvfile:
spamreader=csv.reader(csvfile,delimiter=',')
spamreader.next()
for row in spamreader:
XDevices[NumRows,0]=DictHandle[row[0]]
XDevices[NumRows,1]=DictDevice[row[1]]
XDevices[NumRows,2]=DictDevType[row[2]]
XDevices[NumRows,3]=DictDevOs[row[3]]
XDevices[NumRows,4]=DictCountry[row[4]]
XDevices[NumRows,5]=np.float_(row[5])
XDevices[NumRows,6]=DictAnnC1[row[6]]
XDevices[NumRows,7]=DictAnnC2[row[7]]
XDevices[NumRows,8]=np.float_(row[8])
XDevices[NumRows,9]=np.float_(row[9])
XDevices[NumRows,10]=np.float_(row[10])
NumRows = NumRows + 1
return XDevices
def readSnap(self,f):
snap = Snap()
snap.time = 0
snap.box = []
snap.atoms = []
snap.natoms = 0
for i, line in enumerate(f):
if i > 8 and i < 8 + snap.natoms:
snap.atoms.append(line.split())
elif i == 3:
snap.natoms = int(line.split()[0])
elif i == 5 or i == 6 or i == 7:
snap.box.append(np.float_(line.split()))
elif i == 4:
if len(line.split()) == 3:
snap.boundary = []
else:
snap.boundary = line.split()[3:]
elif i == 8:
snap.descriptor = line.split()[2:]
elif i == 1:
snap.time = int(line.split()[0])
elif i == 8 + snap.natoms:
snap.atoms.append(line.split())
break
snap.atoms = np.float_(snap.atoms)
snap.box = np.array(snap.box)
return snap
def rotatev_aroundz(p,a): #
x=p[0]
y=p[1]
z=p[2]
cs=np.float_(np.cos(a))
cn=np.float_(np.sin(a))
return np.float_(np.array([cs*x-cn*y,cn*x+cs*y,z]))
def compute_edf_distance(support1, support2):
bin_edges = numpy.empty((support1.shape[0] + support2.shape[0] + 2,))
bin_edges[0] = -numpy.inf
bin_edges[-1] = numpy.inf
bin_edges[1 : 1 + support1.shape[0]] = support1
bin_edges[1 + support1.shape[0] : 1 + support1.shape[0] + support2.shape[0]] = support2
bin_edges = numpy.sort(bin_edges)
#print bin_edges.shape
#print bin_edges
bin_edges = numpy.unique(bin_edges)
#bin_edges = get_unique_sorted_array(bin_edges, True)
#print bin_edges.shape
#print bin_edges
#print support1.shape
#print support2.shape
bin_counts1_i,bins1 = numpy.histogram(support1, bin_edges)
bin_counts2_i,bins2 = numpy.histogram(support2, bin_edges)
bin_counts1 = numpy.float_(bin_counts1_i)
bin_counts2 = numpy.float_(bin_counts2_i)
sum_counts1 = numpy.cumsum(bin_counts1) / numpy.sum(bin_counts1)
sum_counts2 = numpy.cumsum(bin_counts2) / numpy.sum(bin_counts2)
delta = numpy.abs(sum_counts1 - sum_counts2)
dist = numpy.max(delta)
return dist
def slidingMax(x,y,dx):
x = np.float_(x)
y = np.float_(y)
LX = len(x)
ymax = np.ones(LX)*y.min()
code=\
"""
int j;
int i;
int j0;
int inloop;
j0=1;
for (i=0; i<LX; i++){
j=j0;
inloop=0;
while ((x(j)<=x(i)+dx/2) && (j<LX) ) {
if ((x(j)>=x(i)-dx/2) && (x(j)<=x(i)+dx/2)) {
if (y(j)>ymax(i)) {
ymax(i) = y(j);
}
inloop=1;
}
if (inloop==0) {
j0=j; // memorize where we started before
}
j++;
}
}
"""
err = weave.inline(code,
['x', 'y', 'dx','LX','ymax'],
type_converters=converters.blitz,
compiler = 'gcc')
return ymax
def check_numpy_scalar_argument_return_generic(self):
f = PyCFunction('foo')
f += Variable('a1', numpy.int_, 'in, out')
f += Variable('a2', numpy.float_, 'in, out')
f += Variable('a3', numpy.complex_, 'in, out')
foo = f.build()
args = 2, 1.2, 1+2j
results = numpy.int_(2), numpy.float_(1.2), numpy.complex(1+2j)
assert_equal(foo(*args),results)
args = [2], [1.2], [1+2j]
assert_equal(foo(*args),results)
args = [2], [1.2], [1,2]
assert_equal(foo(*args),results)
f = PyCFunction('foo')
f += Variable('a1', 'npy_int', 'in, out')
f += Variable('a2', 'npy_float', 'in, out')
f += Variable('a3', 'npy_complex', 'in, out')
foo = f.build()
args = 2, 1.2, 1+2j
results = numpy.int_(2), numpy.float_(1.2), numpy.complex(1+2j)
assert_equal(foo(*args),results)
args = [2], [1.2], [1+2j]
assert_equal(foo(*args),results)
args = [2], [1.2], [1,2]
assert_equal(foo(*args),results)
def from_edf(fname, compression=None, below_water=False, lon=None, lat=None):
"""
DataFrame constructor to open XBT EDF ASCII format.
Examples
--------
>>> from ctd import DataFrame
>>> cast = DataFrame.from_edf('../test/data/XBT.EDF.gz',
... compression='gzip')
>>> fig, ax = cast['temperature'].plot()
>>> ax.axis([20, 24, 19, 0])
>>> ax.grid(True)
"""
f = read_file(fname, compression=compression)
header, names = [], []
for k, line in enumerate(f.readlines()):
line = line.strip()
if line.startswith("Serial Number"):
serial = line.strip().split(":")[1].strip()
elif line.startswith("Latitude"):
hemisphere = line[-1]
lat = line.strip(hemisphere).split(":")[1].strip()
lat = np.float_(lat.split())
if hemisphere == "S":
lat = -(lat[0] + lat[1] / 60.0)
elif hemisphere == "N":
lat = lat[0] + lat[1] / 60.0
else:
raise ValueError("Latitude not recognized.")
elif line.startswith("Longitude"):
hemisphere = line[-1]
lon = line.strip(hemisphere).split(":")[1].strip()
lon = np.float_(lon.split())
if hemisphere == "W":
lon = -(lon[0] + lon[1] / 60.0)
elif hemisphere == "E":
lon = lon[0] + lon[1] / 60.0
else:
raise ValueError("Longitude not recognized.")
else:
header.append(line)
if line.startswith("Field"):
col, unit = [l.strip().lower() for l in line.split(":")]
names.append(unit.split()[0])
if line == "// Data":
skiprows = k + 1
break
f.seek(0)
cast = read_table(
f, header=None, index_col=None, names=names, dtype=float, skiprows=skiprows, delim_whitespace=True
)
f.close()
cast.set_index("depth", drop=True, inplace=True)
cast.index.name = "Depth [m]"
name = basename(fname)[1]
if below_water:
cast = remove_above_water(cast)
return CTD(cast, longitude=lon, latitude=lat, serial=serial, name=name, header=header)
def allan(freq,noise, t):
#first, convert to phase noise
phase=np.float_(noise*1.934*(10**14)/2./(freq**2.))
#now integrate over all values to get allan variance
transferFunc = np.float_(np.sin(np.pi*phase*t)**4/(np.pi*t*phase)**2)
out = 2*integrate.trapz(phase*transferFunc,freq)
return np.sqrt(out)
def smooth_spreadsheet_with_rwr(restart, network_sparse, run_parameters):
""" Simulates a random walk with restarts.
alpha=0.7, max_iteration=100, tol=1.e-4
Args:
restart: restart array of any size.
network_sparse: adjancy matrix stored in sparse format.
run_parameters: parameters dictionary with alpha, restart_tolerance,
number_of_iteriations_in_rwr and
max_iteration: maximum number of random walap_vals. (default = 100)
tol: convergence tolerance. (default = 1.e-4)
Returns:
smooth_1: smoothed restart data.
step: number of iterations used
"""
tol = np.float_(run_parameters["restart_tolerance"])
alpha = np.float_(run_parameters["restart_probability"])
smooth_0 = restart
smooth_r = (1. - alpha) * restart
for step in range(0, int(run_parameters["number_of_iteriations_in_rwr"])):
smooth_1 = alpha * network_sparse.dot(smooth_0) + smooth_r
deltav = LA.norm(smooth_1 - smooth_0, 'fro')
if deltav < tol:
break
smooth_0 = smooth_1
return smooth_1, step
def read_data (self):
"""Reads in data required for the given input_type."""
re_int = '\d+'
re_ints = '\\b\d+\\b'
re_floats = '[+-]? *(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?'
f_name = self.input_path + "l2errs+convergence_standard.txt"
var_names = list()
with open(f_name) as f:
for line in f:
if ('n_vars' in line):
n_vars = np.int_(re.search(re_int,line).group())
if ('ml_max' in line):
ml_max = np.int_(re.search(re_int,line).group())
if ('p_max' in line):
p_max = np.int_(re.search(re_int,line).group())
# Inialize np arrays now that dimensions are known
block_size2 = (ml_max+1,p_max+1)
block_size3 = (ml_max+1,p_max+1,n_vars)
cases_run = np.int_( np.zeros(block_size2))
h = np.float_(np.zeros(block_size2))
l2_errors = np.float_(-float('inf')+np.zeros(block_size3))
conv_orders = np.float_(-float('inf')+np.zeros(block_size3))
if ('Cases Run' in line):
for i in range(0,ml_max+1):
line = f.readline()
cases_run[i,:] = np.int_([int(s) for s in re.findall(re_ints, line)])
if ('Mesh Size' in line):
assign_block(h,cases_run,f)
if ('L2 Errors' in line):
for k in range(0,n_vars):
skip_lines(1,f)
line = f.readline()
var_names.append(line.replace('\n',''))
assign_block(l2_errors[:,:,k],cases_run,f)
if ('Convergence Orders' in line):
for k in range(0,n_vars):
skip_lines(2,f)
assign_block(conv_orders[:,:,k],cases_run,f)
self.n_vars = n_vars
self.ml_max = ml_max
self.p_max = p_max
self.var_names = var_names
self.cases_run = cases_run
self.h = h
self.l2_errors = l2_errors
self.conv_orders = conv_orders
def initialize_constants(self):
"""
Initialize constants that depend on the instance.
"""
# maximum pheromone value
self.PH_MAX = np.float_(self.num_vars / (1.0 - self.PH_REDUCE_FACTOR))
# minimum pheromone value
self.PH_MIN = np.float_(self.PH_MAX / self.num_lits)
def gray2real01(img):
x = img.shape[0]
y = img.shape[1]
fxy = np.zeros((x,y),dtype=np.float_)
for i in xrange(0,x):
for j in xrange(0,y):
fxy[i,j] = np.float_(img[i][j])/np.float_(255)
return fxy
def __init__(self, length=0, angle=0, order=1, rotate=0, name=None, **kwargs):
self.name = name
self._type = 'bend'
self._length = _np.float_(length)
self._order = int(order)
self._angle = _np.float_(angle)
self.rotate = rotate
self._kwargs = kwargs
def snack_raw_pitch_python(wav_fn, frame_shift, window_size, max_pitch, min_pitch):
"""Implement snack_raw_pitch() by calling Snack through Python's tkinter
library
Note this method can only be used if the user's machine is setup, so that
Tcl/Tk can be accessed through Python's tkinter library.
The vectors returned here are the raw Snack output, without padding.
For more info, see documentation for snack_raw_pitch().
"""
try:
import tkinter
except ImportError:
try:
import Tkinter as tkinter
except ImportError: # pragma: no cover
print("Need Python library tkinter. Is it installed?")
# HACK: Need to replace single backslash with two backslashes,
# so that the Tcl shell reads the file path correctly on Windows
if sys.platform == 'win32' or sys.platform == 'cygwin': # pragma: no cover
wav_fn = wav_fn.replace('\\', '\\\\')
# XXX I'm assuming Hz for pitch; the docs don't actually say that.
# http://www.speech.kth.se/snack/man/snack2.2/tcl-man.html#spitch
tcl = tkinter.Tcl()
try:
# XXX This will trigger a message 'cannot open /dev/mixer' on the
# console if you don't have a /dev/mixer. You don't *need* a mixer to
# snack the way we are using it, but there's no practical way to
# suppress the message without modifying the snack source. Fortunately
# most people running opensauce will in fact have a /dev/mixer.
tcl.eval('package require snack')
except tkinter.TclError as err: # pragma: no cover
log.critical('Cannot load snack (is it installed?): %s', err)
return
tcl.eval('snack::sound s')
tcl.eval('s read {}'.format(wav_fn))
cmd = ['s pitch -method esps']
cmd.extend(['-framelength {}'.format(frame_shift / 1000)])
cmd.extend(['-windowlength {}'.format(window_size / 1000)])
cmd.extend(['-maxpitch {}'.format(max_pitch)])
cmd.extend(['-minpitch {}'.format(min_pitch)])
# Run Snack pitch command
tcl.eval('set data [{}]'.format(' '.join(cmd)))
# XXX check for errors here and log and abort if there is one. Result
# string will start with ERROR:.
# Collect results and save in return variables
num_frames = int(tcl.eval('llength $data'))
F0_raw = np.empty(num_frames)
V_raw = np.empty(num_frames)
# snack returns four values per frame, we only care about the first two.
for i in range(num_frames):
values = tcl.eval('lindex $data ' + str(i)).split()
F0_raw[i] = np.float_(values[0])
V_raw[i] = np.float_(values[1])
return F0_raw, V_raw
def parseProcess():
global trainData, trainLabel, testData, testLabel
trainData = np.array([],dtype=float)
procInfoHex = []
#trainLabel = np.array([],dtype=int)
testData = np.array([], dtype=float)
#testLabel = np.array([], dtype=int)
#entry[0] = ['USER', 'PID', '%CPU', '%MEM', 'VSZ', 'RSS', 'TTY', 'STAT', 'START', 'TIME', 'COMMAND']
#USER(0), %CPU(2), %MEM(3), VSZ(4), RSS(5), TIME(9), CMD(10) are important attribute
init = 1
iteration = 0
while iteration < NUM_ITR: #100 iteration, about 2000 samples
shDump = commands.getoutput("ps aux --sort=-pcpu") # list of all processes in stream
entry = shDump.split('\n') #split the stream into arraies
#print len(entry)
for indx in range(len(entry)):
hexList = []
if indx != 0:
procInfo = entry[indx].split() #split array into elements
#remove meaningless contents
procInfo.pop(1);
procInfo.pop(5);
procInfo.pop(6);
#procInfo = ['USER', '%CPU', '%MEM', 'VSZ', 'RSS','STAT','TIME', 'COMMAND','param0', 'param1', 'param2', 'param3', 'param4', .... 'paramN']
#convert plain text string to 8-byte element a pop
for s in procInfo:
#if an attribute is more than 8 bytes, we split it
procInfoHex = split_hex(s.encode('hex'))
procInfoHex = procInfoHex.split()
hexList= hexList + procInfoHex
#the buffer is 60 doubles wide, so padding the remainders with ' '
while len(hexList) < NUM_COLS:
hexList.append(' '.encode('hex'))
#covert Hex to double
procInfoHash = [ struct.unpack('<d', s.zfill(16).decode('hex'))[0] for s in hexList]
if indx == 1 and init == 1:
init=0 #only init numpy once
#procInfo[7] is the command, use it as label
trainData=np.hstack((trainData, np.float_(procInfoHash)));
#trainLabel=np.append(trainLabel, np.int_(procInfoHash[7]));
else:
trainData=np.vstack((trainData, np.float_(procInfoHash)));
#trainLabel=np.append(trainLabel, np.int_(procInfoHash[7]));
iteration += 1
# Split into train and test
trainData, testData= sklearn.cross_validation.train_test_split(trainData)
def change_E(self, old_gamma, new_gamma):
"""
Scale the setpoint energy of the magnet from the old energy *old_gamma* to new energy *new_gamma*.
The best way to think about this is that the angle will change with different beam energy, so changing the beam energy changes the magnet's properties, because the magnet's B-field stays the same.
"""
old_gamma = _np.float_(old_gamma)
new_gamma = _np.float_(new_gamma)
self._angle *= old_gamma / new_gamma
def norm(data):
n = np.zeros((len(data),1),dtype=np.float_)
for j in range(0,len(data)):
s=np.float_(0.0)
for i in range(0,len(data[0])):
s+=data[j,i]*data[j,i]
n[j,0] = np.float_(1/np.sqrt(s))
# return (np.array(data,dtype=np.float_)*n)
return (n)
def __init__(self, timber_variable, press_gauge):
if type(timber_variable) is str:
dict_timber = tm.parse_timber_file(timber_variable, verbose=True)
timber_press_gauge = dict_timber[press_gauge]
elif type(timber_variable) is dict:
timber_press_gauge = timber_variable[press_gauge]
self.t_stamps = np.float_(np.array(timber_press_gauge.t_stamps))
self.values = np.squeeze(np.float_(np.array(timber_press_gauge.values)))
def pvals_logRatioMC(counts1, counts2, B=1e6, pseudocount=1, verbose=False):
"""Compute component-wise p-values of difference between two count vectors
using Monte Carlo sampling of log ratios.
Null hypothesis is that data is from same multinomial. Parameters estimated
by combining both count vectors. Zeros are handled by adding pseudocount to
each element.
The test statistic is log Ratio, which is computed for each component.
Two random count vectors are generated, and and component-wise log ratio
is computed. For each component, it is recorded whether the abs random log
ratio was greater than or less than the abs test statistic value. This is
performed B times. The absolute value makes the test two-sided and symmetric.
The achieved significance level (ASL) is returned for each component.
"""
if len(counts1) != len(counts2): raise ValueError, "Counts vectors have different lengths."
counts1 = np.asarray(counts1, dtype=np.float)
counts2 = np.asarray(counts2, dtype=np.float)
total1 = int(np.round(np.sum(counts1)))
total2 = int(np.round(np.sum(counts2)))
countsMLE = counts1 + counts2 + pseudocount
counts1 = counts1 + pseudocount # note: counts1 and counts2 are changed at this point
counts2 = counts2 + pseudocount
normcounts1 = counts1 / np.sum(counts1)
normcounts2 = counts2 / np.sum(counts2)
testabslogratios = np.abs(np.log10(normcounts2 / normcounts1))
probvec = countsMLE / np.sum(countsMLE)
atleastasextreme = np.zeros(len(counts1))
for i in xrange(B):
if verbose and i % 10 == 0:
sys.stdout.write("%i " % i)
sys.stdout.flush()
randcounts1 = np.float_(np.random.multinomial(total1, probvec)) + pseudocount
randcounts2 = np.float_(np.random.multinomial(total2, probvec)) + pseudocount
normrandcounts1 = randcounts1 / np.sum(randcounts1)
normrandcounts2 = randcounts2 / np.sum(randcounts2)
randabslogratios = np.abs(np.log10(normrandcounts2 / normrandcounts1))
atleastasextreme += np.float_(randabslogratios >= testabslogratios)
ASL = atleastasextreme / B
return ASL
def sdds_to_dict(in_complete_path):
us_string = in_complete_path.split('.')[-2]
try:
temp = sddsdata(in_complete_path, endian='little', full=True)
except IndexError:
print 'Failed to open data file. (save_bct_mat)'
return
data = temp.data[0]
cycleTime = data['cycleTime'].tostring()
t_stamp_unix = time.mktime(time.strptime(cycleTime.replace('"', '').replace('\n','').split('.')[0], '%Y/%m/%d %H:%M:%S'))
beamID = np.int_(data['beamID'])
deviceName = ((data['deviceName'].tostring()).split('\n')[0]).split('SPS.')[-1]
sbfIntensity = np.float_(data['sbfIntensity'])
acqState = np.int_(data['acqState'])
totalIntensity = data['totalIntensity']
acqTime = data['acqTime'].tostring()
propType = np.int_(data['propType'])
totalIntensity_unitExponent = np.int_(data['totalIntensity_unitExponent'])
measStamp_unitExponent = np.float_(data['measStamp_unitExponent'])
samplingTime = np.int_(data['samplingTime'])
measStamp_unit = np.int_(data['measStamp_unit'])
observables = np.int_(data['observables'])
nbOfMeas = np.int_(data['nbOfMeas'])
superCycleNb = np.int_(data['superCycleNb'])
acqMsg = data['acqMsg'].tostring()
measStamp = data['measStamp']
acqDesc = data['acqDesc'].tostring()
totalIntensity_unit = np.int_(data['totalIntensity_unit'])
dict_meas = {
'beamID':beamID,
'deviceName':deviceName,
'sbfIntensity':sbfIntensity,
'acqState':acqState,
'totalIntensity':totalIntensity,
'acqTime':acqTime,
'propType':propType,
'totalIntensity_unitExponent':totalIntensity_unitExponent,
'measStamp_unitExponent':measStamp_unitExponent,
'samplingTime':samplingTime,
'measStamp_unit':measStamp_unit,
'observables':observables,
'nbOfMeas':nbOfMeas,
'superCycleNb':superCycleNb,
'cycleTime':cycleTime,
'acqMsg':acqMsg,
'measStamp':measStamp,
'acqDesc':acqDesc,
'totalIntensity_unit':totalIntensity_unit,
'SPSuser':us_string,
't_stamp_unix':t_stamp_unix
}
return dict_meas
def __init__(self, timber_variable, beam=0):
if type(timber_variable) is dict:
timber_variable_BCT = timber_variable[get_variable_dict(beam)['BEAM_INTENSITY']]
self.beam=beam
self.t_stamps = np.float_(np.array(timber_variable_BCT[0]))
self.t_str=[datetime.datetime.fromtimestamp(self.t_stamps[ii]) for ii in np.arange(len(self.t_stamps))]
self.values = np.squeeze(np.float_(np.array(timber_variable_BCT[1])))
def quat2rpy2(Q):
w = Q[0]
x = Q[1]
y = Q[2]
z = Q[3]
r = np.float_(np.arctan2(2*y*w - 2*x*z, 1 -2*y*y - 2*z*z))
p = np.float_(np.arctan2(2*x*w - 2*y*z, 1 -2*x*x - 2*z*z))
y = np.float_(np.arcsin(2*x*y + 2*z*w))
return np.float_([r,p,y])
fill_dict.update(tm.CalsVariables_from_h5(
data_folder_fill + ('/fill_basic_data_h5s/'
'basic_data_fill_%d.h5'%filln_offset)))
fill_dict.update(tm.CalsVariables_from_h5(
data_folder_fill + ('/fill_heatload_data_h5s/'
'heatloads_fill_%d.h5'%filln_offset)))
if args.use_recalc:
fill_dict.update(qf.get_fill_dict(filln_offset,
h5_storage=H5_storage(recalc_h5_folder),
use_dP=True))
dict_offsets={}
for kk in hl_varlist:
dict_offsets[kk] = np.interp(t_zero_unix, np.float_(np.array(fill_dict[kk].t_stamps)), fill_dict[kk].float_values())
pl.close('all')
ms.mystyle_arial(fontsz=fontsz, dist_tick_lab=9)
fig = pl.figure(1, figsize=figsz)
fig.patch.set_facecolor('w')
ax1 = fig.add_subplot(311)
ax11 = ax1.twinx()
ax2 = fig.add_subplot(312, sharex=ax1)
ax3 = fig.add_subplot(313, sharex=ax1)
ms.sciy()
N_fills = len(fill_list)
t_for_integrated = []
else:
raise ValueError, "Must give a single argument that is a timeseries data file"
data = timeseries.load_timeseries(inhandle)
labels = data['labels']
times = data['times']
timeseriesmatrix = data['matrix']
try:
sums = data['sums']
except KeyError:
sums = timeseriesmatrix.sum(axis=0)
# normalize if desired
if options.normalize:
timeseriesmatrix = np.float_(timeseriesmatrix) / np.asarray(sums)
# define which time series to plot
if options.threshold:
idxs = np.sum(timeseriesmatrix >= options.threshold,
axis=1) > 0 # breaks threshold at least once
else:
idxs = np.asarray([True] * timeseriesmatrix.shape[0])
# idxs = np.sum(time_series_freqs>0,axis=1)>2 # seen at least twice
# idxs_bool = np.logical_and(idxs_bool_1,idxs_bool_2)
# idxs_bool = np.array([False]*len(reference_clones))
print "Number of lines plotted: %i" % np.sum(idxs)
# ==================
# = Make the plots =
# ==================
def cross_entropy(Y, P):
Y = np.float_(Y)
P = np.float_(P)
return - np.sum(Y * np.log(P) + (1 - Y) * np.log(1 - P))
import glob
import numpy as np
import matplotlib.pyplot as plt
import sklearn.cluster
images = glob.glob("./imagenes/*.png")
data = []
for i in range(len(images)):
d = np.float_(plt.imread(images[i]).flatten())
data.append(d)
data = np.array(data)
# n_cluster_array =
n_clusters = np.arange(1, 21, 1)
inertia_array = []
for nc in n_clusters:
k_means = sklearn.cluster.KMeans(n_clusters=nc)
k_means.fit(data)
# calculo a cual cluster pertenece cada pixel
cluster = k_means.predict(data)
# asigno a cada pixel el lugar del centro de su cluster
data_centered = data.copy()
for i in range(nc):
ii = cluster == i
data_centered[ii, :] = np.int_(k_means.cluster_centers_[i])
def test_recursively_convert_to_json_serializable():
asset = ge.dataset.PandasDataset({
"x": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
})
asset.expect_column_values_to_be_in_set("x", [1, 2, 3, 4, 5, 6, 7, 8, 9],
mostly=0.8)
part = ge.dataset.util.partition_data(asset.x)
asset.expect_column_kl_divergence_to_be_less_than("x", part, 0.6)
# Dumping this JSON object verifies that everything is serializable
json.dumps(expectationSuiteSchema.dump(asset.get_expectation_suite()),
indent=2)
x = {
"w": ["aaaa", "bbbb", 1.3, 5, 6, 7],
"x": np.array([1, 2, 3]),
"y": {
"alpha": None,
"beta": np.nan,
"delta": np.inf,
"gamma": -np.inf
},
"z": {1, 2, 3, 4, 5},
"zz": (1, 2, 3),
"zzz": [
datetime.datetime(2017, 1, 1),
datetime.date(2017, 5, 1),
],
"np.bool": np.bool_([True, False, True]),
"np.int_": np.int_([5, 3, 2]),
"np.int8": np.int8([5, 3, 2]),
"np.int16": np.int16([10, 6, 4]),
"np.int32": np.int32([20, 12, 8]),
"np.uint": np.uint([20, 5, 6]),
"np.uint8": np.uint8([40, 10, 12]),
"np.uint64": np.uint64([80, 20, 24]),
"np.float_": np.float_([3.2, 5.6, 7.8]),
"np.float32": np.float32([5.999999999, 5.6]),
"np.float64": np.float64([5.9999999999999999999, 10.2]),
# 'np.complex64': np.complex64([10.9999999 + 4.9999999j, 11.2+7.3j]),
# 'np.complex128': np.complex128([20.999999999978335216827+10.99999999j, 22.4+14.6j]),
# 'np.complex256': np.complex256([40.99999999 + 20.99999999j, 44.8+29.2j]),
"np.str": np.unicode_(["hello"]),
"yyy": decimal.Decimal(123.456),
}
if platform.system() != "Windows":
x["np.float128"] = np.float128([5.999999999998786324399999999, 20.4])
x = ge.data_asset.util.recursively_convert_to_json_serializable(x)
assert isinstance(x["x"], list)
assert isinstance(x["np.bool"][0], bool)
assert isinstance(x["np.int_"][0], int)
assert isinstance(x["np.int8"][0], int)
assert isinstance(x["np.int16"][0], int)
assert isinstance(x["np.int32"][0], int)
assert isinstance(x["np.uint"][0], int)
assert isinstance(x["np.uint8"][0], int)
assert isinstance(x["np.uint64"][0], int)
assert isinstance(x["np.float32"][0], float)
assert isinstance(x["np.float64"][0], float)
if platform.system() != "Windows":
assert isinstance(x["np.float128"][0], float)
# self.assertEqual(type(x['np.complex64'][0]), complex)
# self.assertEqual(type(x['np.complex128'][0]), complex)
# self.assertEqual(type(x['np.complex256'][0]), complex)
assert isinstance(x["np.float_"][0], float)
# Make sure nothing is going wrong with precision rounding
if platform.system() != "Windows":
assert np.allclose(
x["np.float128"][0],
5.999999999998786324399999999,
atol=10**(-sys.float_info.dig),
)
# TypeError when non-serializable numpy object is in dataset.
with pytest.raises(TypeError):
y = {"p": np.DataSource()}
ge.data_asset.util.recursively_convert_to_json_serializable(y)
try:
x = unicode("abcdefg")
x = ge.data_asset.util.recursively_convert_to_json_serializable(x)
assert isinstance(x, unicode)
except NameError:
pass
# By Luca
#### DEMO
from astropy.io import fits
import numpy as np
import matplotlib.pyplot as plt
from astropy.visualization import make_lupton_rgb
from astropy.visualization import *
### READING
b = fits.open("data/"+"M66-Blue.fts")[0].data
r = fits.open("data/"+"M66-Red.fts")[0].data
g = fits.open("data/"+"M66-Green.fts")[0].data
forCasting = np.float_()
### CASTING
r = np.array(r,forCasting)
g = np.array(g,forCasting)
b = np.array(b,forCasting)
rgb_default = make_lupton_rgb(r, g, b, minimum=1000, stretch=900, Q=100, filename="prova2finale.png")
plt.imshow(rgb_default, origin='lower')
plt.show()
def wfits(name, data, hdr):
if ptt.exists(name) == False:
wfit(name, data, hdr)
else:
sycall("rm " + name)
wfit(name, data, hdr)
sys.argv = filter(None, sys.argv)
if len(sys.argv) < 9:
print "USE: plot_maps.pl map.fits min max bright contrast Label factor dev [MASK_MAP]"
sys.exit(0)
mapfile = sys.argv[1]
min = float_(sys.argv[2])
max = float_(sys.argv[3])
bright = sys.argv[4]
contrast = sys.argv[5]
label = sys.argv[6]
factor = float_(sys.argv[7])
dev = sys.argv[8]
id_mask = 0
if len(sys.argv) == 10:
mask_file = sys.argv[9]
id_mask = 1
print dev
if dev == "null":
import matplotlib.pyplot as plt
# from matplotlib import gridspec
import random
import sys
from six.moves import zip as izip
from six.moves import xrange
from . import _minpack
from . import _minim
from . import _saoopt
from sherpa.utils import parallel_map
from sherpa.utils._utils import sao_fcmp
#
# Use FLT_EPSILON as default tolerance
#
EPSILON = numpy.float_(numpy.finfo(numpy.float32).eps)
#
# Maximum callback function value, used to indicate that the optimizer
# has exceeded parameter boundaries. All the optimizers expect double
# precision arguments, so we use numpy.float_ instead of SherpaFloat.
#
# As of numpy 0.9.5, 'max' is a 0-D array, which seems like a bug.
#
FUNC_MAX = numpy.float_(numpy.finfo(numpy.float_).max)
def _check_args(x0, xmin, xmax):
x = numpy.array(x0, numpy.float_) # Make a copy
xmin = numpy.asarray(xmin, numpy.float_)
xmax = numpy.asarray(xmax, numpy.float_)
#
# print(r_eci)
# print(v_eci)
json_list = []
print('Parsing file and converting to ECI...')
with open(sys.argv[1], "r") as f:
for line in f:
temp = line.split()
epoch_utc = Time(temp[4], scale='utc', format='isot')
v = CartesianDifferential(list(np.float_(temp[17:20]))*(u.m/u.s))
r = CartesianRepresentation(list(np.float_(temp[14:17]))*u.m, differentials={'s': v})
r_ecef = ITRS(r, obstime=epoch_utc)
r_eci_temp = r_ecef.transform_to(GCRS(obstime=epoch_utc))
v_eci = r_eci_temp.cartesian.differentials['s'].d_xyz # this may be off from Jonathan's initial notebook test
r_eci = r_eci_temp.cartesian.xyz
utc_ms = temp[4][:-3] # FIXME -- orbdetpy can handle 6 digits of precision in time so need to remove this
json_object = {"Time": utc_ms + "Z", # add this to maintain ISO8601 format for orbdetpy
"PositionVelocity": [
r_eci[0].value,
r_eci[1].value,
r_eci[2].value,
v_eci[0].value*1000, # not sure why this changed to km/s
v_eci[1].value*1000,
v_eci[2].value*1000
ret2, labels2, stats2, centroids2 = cv2.connectedComponentsWithStats(dst2)
criteria2 = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 100, 0.001)
corners2 = cv2.cornerSubPix(imagem2,np.float32(centroids2),(5,5),(-1,-1),criteria2)
matriz2=[]
for variavel in corners2:
array=np.array([variavel])
matriz2.append(array)
kp2 = cv2.KeyPoint_convert(matriz2)
kp2, des2 = brief.compute(img2, kp2)
mat = bf.match(des1,des2)
mat = sorted(mat, key = lambda x:x.distance)
matches = mat[0:150]
with open("../../imgTeste/img"+str(quantidadeImagens)+".txt",'r') as f:
texto=f.readlines()
posicao_x= np.float_(texto[0:4])
posicao_y = np.float_(texto[4:8])
min_x = float(min(posicao_x))
max_x = float(max(posicao_x))
min_y = float(min(posicao_y))
max_y = float(max(posicao_y))
if len(matches)>10:
src_pts = np.float32([ kp1[m.queryIdx].pt for m in matches ]).reshape(-1,1,2)
dst_pts = np.float32([ kp2[m.trainIdx].pt for m in matches ]).reshape(-1,1,2)
M, mask = cv2.findHomography(src_pts, dst_pts, cv2.RANSAC, 5.0)
h,w = img1.shape
pts = np.float32([ [0,0],[0,h-1],[w-1,h-1],[w-1,0] ]).reshape(-1,1,2)
dst_comp = cv2.perspectiveTransform(pts,M)
img2 = cv2.polylines(img2,[np.int32(dst_comp)],True,255,3, cv2.LINE_AA)
for pos in dst_pts:
sycall("rm " + name1)
wfit(name, data, hdr)
sys.argv = filter(None, sys.argv)
if len(sys.argv) < 5:
print "USE: spec_cube_plot.py INPUT_FILE.CUBE.FITS SN_map.fits signalmap.fits noisemap.fits [WMIN,WMAX]"
sys.exit(0)
input = sys.argv[1]
sn_map = sys.argv[2]
signal_map = sys.argv[3]
noise_map = sys.argv[4]
if len(sys.argv) == 6:
data = sys.argv[5].split(',')
wmin = float_(data[0])
wmax = float_(data[1])
else:
wmin = 0.0
wmax = 0.0
y_min = 1e12
y_max = -1e12
n = 0
[pdl, hdr] = gdata(input, 0, header=True)
[nz, nx, ny] = pdl.shape
crval = hdr["CRVAL3"]
cdelt = hdr["CDELT3"]
crpix = hdr["CRPIX3"]
if cdelt == 0:
def to_float(_int):
try:
return np.float_(_int)
except OverflowError:
return np.nan
def resize_array_example(Array_in, Array_example, method=1):
"""
This function resizes an array so it has the same size as an example array
The extend of the array must be the same
Keyword arguments:
Array_in -- []
Array: 2D or 3D array
Array_example -- []
Array: 2D or 3D array
method: -- 1 ... 5
int: Resampling method
"""
# Create old raster
Array_out_shape = np.int_(Array_in.shape)
Array_out_shape[-1] = Array_example.shape[-1]
Array_out_shape[-2] = Array_example.shape[-2]
if method == 1:
interpolation_method = 'nearest'
interpolation_number = 0
if method == 2:
interpolation_method = 'bicubic'
interpolation_number = 3
if method == 3:
interpolation_method = 'bilinear'
interpolation_number = 1
if method == 4:
interpolation_method = 'cubic'
if method == 5:
interpolation_method = 'lanczos'
if len(Array_out_shape) == 3:
Array_out = np.zeros(Array_out_shape)
for i in range(0, Array_out_shape[0]):
Array_in_slice = Array_in[i, :, :]
size = tuple(Array_out_shape[1:])
if sys.version_info[0] == 2:
import scipy.misc as misc
Array_out_slice = misc.imresize(np.float_(Array_in_slice),
size,
interp=interpolation_method,
mode='F')
if sys.version_info[0] == 3:
import skimage.transform as transform
Array_out_slice = transform.resize(np.float_(Array_in_slice),
size,
order=interpolation_number)
Array_out[i, :, :] = Array_out_slice
elif len(Array_out_shape) == 2:
size = tuple(Array_out_shape)
if sys.version_info[0] == 2:
import scipy.misc as misc
Array_out = misc.imresize(np.float_(Array_in),
size,
interp=interpolation_method,
mode='F')
if sys.version_info[0] == 3:
import skimage.transform as transform
Array_out = transform.resize(np.float_(Array_in),
size,
order=interpolation_number)
else:
print('only 2D or 3D dimensions are supported')
return (Array_out)
pres = f.variables['ps'][:] * 1e-2 #[:ix,...]*1e-2 # Converting to hPa
# pres=f.variables['ps'][:ix,...]*1e-2 # Converting to hPa
f.close()
### Some specific humidity values are < 0,
### possibly an interpolation issue.
q[q < 0] = 0.0
qps[qps < 0] = 0.0
pbl_top = pres - 100 ## The sub-cloud layer is 100 mb thick ##
low_top = np.zeros_like(pres)
low_top[:] = 500
# low_top=pbl_top-400 ## The next layer extends to 500 mb ##
pbl_top = np.float_(pbl_top.flatten())
low_top = np.float_(low_top.flatten())
lev = np.float_(lev)
print('PREPPING LAYER AVERAGING-2')
pbl_ind = np.zeros(pbl_top.size, dtype=np.int64)
low_ind = np.zeros(low_top.size, dtype=np.int64)
find_closest_index(pbl_top, lev, pbl_ind)
find_closest_index(low_top, lev, low_ind)
pres_3d = np.zeros_like(t)
pres_3d[:] = pres[:, None, :, :]
levels = np.zeros_like(t)
def __init__(self, wave, flux, error=None):
#Could be np.asarray, but change fit_spectral_lines.py
self.wave = np.float_(wave)
self.flux = np.float_(flux)
self.error = np.float_(error)
def __init__(self, delta_t_recent, leap_dates, leap_offsets):
self.delta_t_table = build_delta_t_table(delta_t_recent)
self.leap_dates, self.leap_offsets = leap_dates, leap_offsets
self.J2000 = Time(self, float_(T0))
self.B1950 = Time(self, float_(B1950))
def error_formula(y, y_hat):
y = np.float_(y)
y_hat = np.float_(y_hat)
return -np.sum(y * np.log(y_hat) + (1 - y) * np.log(1 - y_hat))
# CF_score_dict = {} # score of CF
combined_score = {} # ensemble score...
for i in range(len(triplets)):
score_dict[triplets[i,2]] = score[i][0]
# for idx in del_idx:
# del score_dict[idx]
# get score from PaperRank here...
PaperRank_score = all_PaperRank_score[p]
for cited_paper in score_dict.keys():
if str(cited_paper) not in PaperRank_score:
PaperRank_score[str(cited_paper)] = 0.0
# print('len after adding cited paper:', len(PaperRank_score))
PaperRank_score_val = list(np.float_(list(PaperRank_score.values()))) # list(np.float_(list_name))
PaperRank_score_key = list(PaperRank_score.keys())
PaperRank_score_normalized_val = (PaperRank_score_val - np.min(PaperRank_score_val)) / np.ptp(PaperRank_score_val) if np.ptp(PaperRank_score_val) != 0 else [0]*len(PaperRank_score_val)# score of PaperRank is normalized here...
for i in range(0,len(PaperRank_score)):
PaperRank_score[PaperRank_score_key[i]] = PaperRank_score_normalized_val[i]
# print(PaperRank_score)
# print('final len:',len(PaperRank_score_val))
# exit(0)
# calculate ensemble score here (alpha*ConvCN + (1-alpha)*CF)
# loop through score_dict
for cited_paper in score_dict.keys():
# print('ConvCN score:',score_dict[cited_paper], 'CF score', CF_score.get(str(cited_paper),0))
def center(X):
np.array([np.float_(x) for x in X.T])
np.array([np.float_(1) for _ in X.T])
return X
def _decode(cls, outputs):
return np.float_(outputs[cls.out_t])
def fit_spec(spec,
faxis,
Jupp=12,
K_fit=7,
cutoff=0.009,
varyf=2,
interactive=True,
mode='single'):
"""
Fit the hyperfine lines of CH3CN, derive best-fitted Trot and Ntot.
Input:
spec: the spectra
faxis: the frequency axis
Jupp: J of the upper level
K_fit: the number of K transitions to fit (e.g. K=0-6, then K_fit=7)
cutoff: not used...
varyf: number of channels to vary after you select the Vlsr
interactive: true or false
mode: single or double, components along the line of sight (to be done...)
"""
# Define the J, K numbers as global variables:
# (Not recommended by Python experts...)
global J
global Kladder
J = Jupp
Kladder = K_fit
if interactive:
plt.ion()
f = plt.figure(figsize=(14, 8))
ax = f.add_subplot(111)
unsatisfied = True
while unsatisfied:
if interactive:
f.clear()
plt.ion()
plt.plot(faxis, spec, 'k-', label='Spectrum')
cutoff_line = [cutoff] * len(faxis)
cutoff_line_minus = [-1.0 * cutoff] * len(faxis)
plt.plot(faxis, cutoff_line, 'r-')
plt.plot(faxis, cutoff_line_minus, 'r-')
plt.xlabel(r'Sky Frequency (GHz)', fontsize=20, labelpad=20)
plt.ylabel(r'$T_{\nu}$ (K)', fontsize=20)
plt.text(0.02,
0.92,
sourcename,
transform=ax.transAxes,
color='r',
fontsize=15)
#plt.ylim([-10,60])
#clickvalue = []
if mode == 'single':
cid = f.canvas.mpl_connect('button_press_event', onclick)
raw_input('Click on the plot to select a Vlsr...')
#print clickvalue
if len(clickvalue) >= 1:
print 'Please select at least one velocity! The last one will be used.'
vlsr1 = clickvalue[-1]
vlsr1 = c * (1 - vlsr1 / ch3cn_info['frest'][0]) / 1e5
elif len(clickvalue) == 0:
vlsr1 = 0.0
print 'Or input one velocity manually:'
manualv = raw_input()
manualv = manualv.split()
if len(manualv) == 1:
vlsr1 = np.float_(manualv)
else:
print 'Invalid input...'
print 'The Vlsr is %0.2f km/s' % vlsr1
raw_input('Press any key to start fitting...')
f.canvas.mpl_disconnect(cid)
vlsr2 = 0.0
elif mode == 'double':
cid = f.canvas.mpl_connect('button_press_event', onclick)
raw_input('Click on the plot to select Vlsrs...')
print clickvalue
if len(clickvalue) >= 2:
print 'Please select at least two velocities! The last two will be used.'
vlsr1, vlsr2 = clickvalue[-2], clickvalue[-1]
elif len(clickvalue) == 1:
vlsr1 = clickvalue[-1]
vlsr2 = 0.0
elif len(clickvalue) == 0:
vlsr1, vlsr2 = 0.0, 0.0
print 'Or input two velocities manually:'
manualv = raw_input()
manualv = manualv.split()
if len(manualv) == 2:
vlsr1, vlsr2 = np.float_(manualv)
else:
print 'Invalid input...'
print 'The two Vlsrs are %0.2f km/s and %0.2f km/s.' % (vlsr1,
vlsr2)
raw_input('Press any key to start fitting...')
f.canvas.mpl_disconnect(cid)
else:
vlsr1, vlsr2 = 0.0, 0.0
else:
if mode == 'single':
if spec_low.max() >= cutoff:
print 'Reserved space...'
else:
vlsr1 = 0.0
vlsr2 = 0.0
elif mode == 'double':
vlsr1, vlsr2 = 86.0, 88.0
else:
vlsr1, vlsr2 = 0.0, 0.0
plt.text(0.02,
0.85,
r'V$_{lsr}$=%.1f km/s' % vlsr1,
transform=ax.transAxes,
color='r',
fontsize=15)
fsky_init = ch3cn_info['frest'][0] * (1 - vlsr1 * 1e5 / c)
# Add 4 parameters:
params = Parameters()
if vlsr1 != 0:
params.add('Ntot', value=1e15, min=0, max=1e25)
params.add('T', value=100, min=10)
#params.add('sigma', value=0.0035, vary=False)
params.add('sigma', value=0.0027, min=0, max=0.04)
if varyf > 0:
params.add('fsky', value=fsky_init, min=fsky_init-varyf*chanwidth, \
max=fsky0_init+varyf*chanwidth)
elif varyf == 0:
params.add('fsky', value=fsky_init, vary=False)
if vlsr2 != 0:
print 'Reserved for two-component fitting.'
# Run the non-linear minimization:
if vlsr1 != 0 and vlsr2 != 0:
result = minimize(__model_11_2c__, params, args=(faxis, spec))
elif vlsr1 != 0 or vlsr2 != 0:
result = minimize(__model_11__, params, args=(faxis, spec))
else:
unsatisfied = False
continue
final = spec + result.residual
#report_fit(params)
if interactive:
plt.plot(faxis, final, 'r', label='Best-fitted model')
if vlsr1 != 0 and vlsr2 != 0:
print 'Reserved for two-component fitting.'
elif vlsr1 != 0 or vlsr2 != 0:
plt.text(0.02,
0.80,
r'T$_{rot}$=%.1f($\pm$%.1f) K' %
(result.params['T'].value, result.params['T'].stderr),
transform=ax.transAxes,
color='r',
fontsize=15)
plt.text(0.02,
0.75,
r'N$_{tot}$=%.2e($\pm$%.2e) cm$^{-2}$' %
(result.params['Ntot'].value,
result.params['Ntot'].stderr),
transform=ax.transAxes,
color='r',
fontsize=15)
plt.text(
0.02,
0.70,
r'FWHM=%.2f($\pm$%.2f) km/s' %
(c * result.params['sigma'].value / ch3cn_info['frest'][0]
/ 1e5 * 2.355, c * result.params['sigma'].stderr /
ch3cn_info['frest'][0] / 1e5 * 2.355),
transform=ax.transAxes,
color='r',
fontsize=15)
plt.text(0.02,
0.65,
r'$\tau_\mathrm{K=0}$=%.1e' %
(__tau__(result.params['Ntot'].value,
result.params['sigma'].value,
result.params['T'].value, 0)),
transform=ax.transAxes,
color='r',
fontsize=15)
plt.legend()
plt.draw()
print 'Is the fitting ok? y/n'
yn = raw_input()
if yn == 'y':
unsatisfied = False
currentT = time.strftime("%Y-%m-%d_%H:%M:%S")
plt.savefig('CH3CN_fitting_' + currentT + '.png')
else:
unsatisfied = True
#raw_input('Press any key to continue...')
f.clear()
else:
unsatisfied = False
def get_estimator_boundingbox(tiff_file,
channel=0,
outputdir='.',
w0=1,
w1=100,
h0=10,
h1=1000,
acut=0.9,
aratio_min=2.,
aratio_max=100.,
border_pad=5,
emin=1.0e-4,
debug=False,
threshold=None):
"""
INPUT:
* file to a tiff image.
* (w0,w1): minimum and maximum width for bounding box in pixels.
* (l0,l1): minimum and maximum length for bounding box in pixels.
* acut: minimum area/rectangle bounding box ratio.
* threshold is a lower threshold (everything below is set to zero). Value must be a float between 0 and 1. 1 is the maximum, eg. 255 or 65535.
OUTPUT:
* 2D matrix of weights corresponding to the probability that a pixel belongs to a cell.
USEFUL DOCUMENTATION:
* https://docs.opencv.org/3.4.3/dd/d49/tutorial_py_contour_features.html
* https://docs.opencv.org/2.4/modules/imgproc/doc/filtering.html?highlight=blur#blur
* https://docs.opencv.org/3.1.0/da/d22/tutorial_py_canny.html
* https://docs.opencv.org/3.4/d7/d4d/tutorial_py_thresholding.html
* https://docs.opencv.org/3.4/db/d5c/tutorial_py_bg_subtraction.html
"""
bname = os.path.splitext(os.path.basename(tiff_file))[0]
## read the input tiff_file
img = get_tiff2ndarray(tiff_file, channel=channel)
#img0 = np.copy(img)
# rescale dynamic range linearly (important for OTSU)
amin = np.min(img)
amax = np.max(img)
print "amin = {:.1g} amax = {:.1g}".format(amin, amax)
img = (np.float_(img) - amin) / (amax - amin)
## thresholding to binary mask
norm8 = float(2**8 - 1)
norm16 = float(2**16 - 1)
if threshold is None:
print "OTSU thresholding"
# convert to 8-bit image (Open CV requirement for OTSU)
img = np.array(255 * img, np.uint8)
img8 = np.copy(img)
# OTSU threshold
ret, img = cv2.threshold(img, 0, 255,
cv2.THRESH_BINARY + cv2.THRESH_OTSU)
thres1 = float(ret) / norm8 * (amax - amin) + amin
thres8 = np.uint8(thres1 * norm8)
thres16 = np.uint16(thres1 * norm16)
else:
thres1 = threshold
thres8 = np.uint8(thres1 * norm8)
thres16 = np.uint16(thres1 * norm16)
ret = max((thres1 - amin) / (amax - amin), 0) # value in rescaled DNR
ret = np.uint8(255 * ret) # uint8
print "thres1 = {:.1g} threshold_rescaled_DNR_uint8 = {:d}".format(
thres1, ret)
# convert to 8-bit image (Open CV requirement for OTSU)
img = np.array(255 * img, np.uint8)
img8 = np.copy(img)
# input threshold
ret1, img = cv2.threshold(img, ret, 255, cv2.THRESH_BINARY)
print "thres1 = {:.1g} thres8 = {:d} thres16 = {:d}".format(
thres1, thres8, thres16)
img_bin = np.copy(img)
## opening/closing operations
kernel = np.ones((3, 3), np.uint8) # smoothing kernel
img = cv2.morphologyEx(img, cv2.MORPH_OPEN, kernel)
#img_opening = np.copy(img)
img = cv2.erode(img, kernel, iterations=1)
#img_closing = np.copy(img)
img = cv2.dilate(img, kernel, iterations=1)
img_morph = np.copy(img)
## find connected components
ncomp, labels = cv2.connectedComponents(img)
print "Found {:d} objects".format(ncomp)
## compute the bounding box for the identified labels
#for n in range(ncomp):
height, width = img.shape
Y, X = np.mgrid[0:height, 0:width]
boundingboxes = []
boundingboxes_upright = []
pointsperbox = []
for n in np.arange(ncomp):
idx = (labels == n)
# pixels coordinates for the label
points = np.transpose([X[idx], Y[idx]])
pointsperbox.append(len(points))
# upright rectangles
bb = cv2.boundingRect(points)
boundingboxes_upright.append(bb)
# rotated rectangles
bb = cv2.minAreaRect(points)
boundingboxes.append(bb)
# estimator matrix
## compute scores
scores = []
for n in np.arange(ncomp):
bb = boundingboxes[n]
bb_upright = boundingboxes_upright[n]
area = pointsperbox[n]
# get bounding box width and height
xymid, wh, angle = bb
w, h = wh
if w > h:
wtp = w
w = h
h = wtp
area_rect = w * h
aval = area / area_rect
aratio = h / w
# print "w={:.1f} h={:.1f} aval={:.2e} aratio={:.1f}".format(w,h,aval,aratio)
# computing score
score = 1.
score *= min(1, np.exp((w - w0))) # penalize w < w0
score *= min(1, np.exp(w1 - w)) # penalize w > w1
score *= min(1, np.exp((h - h0))) # penalize w < w0
score *= min(1, np.exp(h1 - h)) # penalize w > w1
score *= min(1, np.exp(aval - acut)) # penalize area/rect < acut
score *= min(1, np.exp(aratio -
aratio_min)) # penalize aratio < aratio_min
score *= min(1, np.exp(aratio_max -
aratio)) # penalize aratio > aratio_max
# check that box corners of an upright bounding box are within the image plus some pad
x, y, ww, hh = bb_upright
x0 = x - border_pad
y0 = y - border_pad
x1 = x + ww + border_pad
y1 = y + hh + border_pad
# print "a0 = {:d} b0 = {:d}".format(x,y)
# print "a1 = {:d} b1 = {:d}".format(x+ww,y+hh)
# print "x0 = {:d} y0 = {:d}".format(x0,y0)
# print "x1 = {:d} y1 = {:d}".format(x1,y1)
if not (x0 >= 0 and x1 < width and y0 >= 0 and y1 < height):
score = 0.
# discard small values
if score < emin:
score = 0.
scores.append(score)
# estimator matrix
eimg = np.zeros(img.shape, dtype=np.float_)
for n in np.arange(ncomp):
idx = (labels == n)
eimg[idx] = scores[n]
nz = np.sum(eimg > 0.)
ntot = len(np.ravel(eimg))
print "nz = {:d} / {:d} sparcity index = {:.2e}".format(
nz, ntot,
float(nz) / float(ntot))
efname = bname
#efile = os.path.join(outputdir,efname+'.txt')
#efile = os.path.join(outputdir,efname+'.pkl')
efile = os.path.join(outputdir, efname + '.npz')
with open(efile, 'w') as fout:
#np.savetxt(fout, eimg)
#pkl.dump(eimg,fout)
ssp.save_npz(efile, ssp.coo_matrix(eimg), compressed=False)
print "{:<20s}{:<s}".format('est. file', efile)
if debug:
debugdir = os.path.join(outputdir, 'debug')
if not os.path.isdir(debugdir):
os.makedirs(debugdir)
# plots
import matplotlib.pyplot as plt
import matplotlib.patches
from matplotlib.path import Path
import matplotlib.collections
from matplotlib.gridspec import GridSpec
## rescale dynamic range linearly
img_base = np.array(img8, dtype=np.float_) / 255.
img_base = (img_base - np.min(img_base)) / (np.max(img_base) -
np.min(img_base))
img_base = np.array(img_base * 255, dtype=np.uint8)
ncolors = (20 - 1)
labels_iterated = np.uint8(labels -
np.int_(labels / ncolors) * ncolors) + 1
images = [img_base, img_bin, img_morph, labels_iterated, eimg]
titles = [
'original', 'binary (thres = {:d})'.format(thres8),
'closing/opening', 'bounding box', 'estimator'
]
cmaps = ['gray', 'gray', 'gray', 'tab20c', 'viridis']
nfig = len(images)
nrow = int(np.ceil(np.sqrt(nfig)))
ncol = nfig / nrow
if (ncol * nrow < nfig): ncol += 1
fig = plt.figure(num=None, figsize=(ncol * 4, nrow * 3))
gs = GridSpec(nrow, ncol, figure=fig)
axes = []
for r in range(nrow):
for c in range(ncol):
ind = r * ncol + c
if not (ind < nfig):
break
ax = fig.add_subplot(gs[r, c])
axes.append(ax)
ax.set_title(titles[ind].upper())
cf = ax.imshow(images[ind], cmap=cmaps[ind])
ax.set_xticks([]), ax.set_yticks([])
if titles[ind] == 'bounding box':
# draw bounding boxes
rects = []
rects_upright = []
codes = [
Path.MOVETO, Path.LINETO, Path.LINETO, Path.LINETO,
Path.CLOSEPOLY
]
for n in range(ncomp):
bb_upright = boundingboxes_upright[n]
bb = boundingboxes[n]
# upright rect
xlo, ylo, w, h = bb_upright
rect = matplotlib.patches.Rectangle((xlo, ylo),
width=w,
height=h,
fill=False)
rects_upright.append(rect)
# rect
verts = cv2.boxPoints(bb)
verts = np.concatenate((verts, [verts[0]]))
path = Path(verts, codes)
rect = matplotlib.patches.PathPatch(path)
rects.append(rect)
col = matplotlib.collections.PatchCollection(
rects_upright,
edgecolors='k',
facecolors='none',
linewidths=0.5)
ax.add_collection(col)
col = matplotlib.collections.PatchCollection(
rects,
edgecolors='r',
facecolors='none',
linewidths=0.5)
ax.add_collection(col)
if titles[ind] == 'estimator':
fig.colorbar(cf, ax=ax)
fname = "{}_estimator_debug".format(bname)
fileout = os.path.join(debugdir, fname + '.png')
gs.tight_layout(fig, w_pad=0)
plt.savefig(fileout, dpi=300)
print "{:<20s}{:<s}".format('debug file', fileout)
plt.close('all')
# from agarpad code: end """
""" canny: start
#test canny filtering
print np.min(img), np.max(img)
maxthres=2500
minthres=100
edges = cv2.Canny(img,threshold1=minthres,threshold2=maxthres,apertureSize=5)
print np.unique(edges)
print "START TEST"
import matplotlib.pyplot as plt
fig = plt.figure(num=None,figsize=(8,4))
plt.subplot(121)
plt.imshow(img, cmap='gray')
plt.xticks([]),plt.yticks([])
plt.subplot(122)
plt.imshow(edges, cmap='gray')
plt.xticks([]),plt.yticks([])
fileout = os.path.join(os.getcwd(),'test_get_estimator_contours.png')
fig.tight_layout()
plt.savefig(fileout,dpi=300)
print "writing: ", fileout
plt.close('all')
fig = plt.figure(num=None,figsize=(4,4))
hist,hedges = np.histogram(np.ravel(img), bins='auto')
plt.bar(hedges[:-1], hist, np.diff(hedges), facecolor='blue', lw=0)
fileout = os.path.join(os.getcwd(),'test_get_estimator_contours_histogram.png')
fig.tight_layout()
plt.savefig(fileout,dpi=300)
print "writing: ", fileout
plt.close('all')
sys.exit()
print "END TEST"
#test
# canny: start """
return os.path.realpath(efile)
def create_scaled_direct_coordinates(xyz, resiprocal_k, k):
out = np.float_(k) * np.einsum(
'ijk,lk->ijl', xyz * 10.0,
resiprocal_k) # "10" means to convert nm to angstrom
return np.mod(out, np.float_(k))
atomtype_form[iatype], dq_form = read_aff(atype_name[iatype])
elif "cm" in args.aff:
atomtype_form[iatype], dq_form = gen_aff_cm(
atype_name[iatype], args.cm_endq, args.cm_dq)
if dq_form != check_dq_form:
raise IOError(
" inconsistent form factor dq's {} for {}-th atomtype (name {}) (ref. initial dq's {})"
.format(dq_form, iatype, str(atype_name[iatype]),
check_dq_form))
# count number of atoms with respect to atom types (to do normalization of s(q))
# writeen by TingTing and edited by Hyuntae
natom_type_kind = np.empty(n_atom_type, dtype=np.int_)
for i_type in range(n_atom_type):
natom_type_kind[i_type] = len(np.where(alist == atype_name[i_type])[0])
frac_natom_type_kind = np.float_(natom_type_kind) / np.float_(
n_atoms_wo_drudes)
print("Done: reading form factor files and make atom type list")
##########################################################################
#***********************************************
# this function calculates B_splines which are used in pme as interpolating
# functions. B_splines are calculated recursively, and therefore it's a good idea
# to grid them
#************************************************
def b_spline(u, n):
# define m2 for n-1 values
mn = np.zeros((n, n))
for i in range(1, n):
ui = u - np.float_(i - 1)
import pandas as pd
import numpy as np
if __name__ == '__main__':
# First, read football.dat into a pandas dataframe
# using any contiguous block of whitespace as the delimiter between columns.
# Upon observing the data, I see that there is a column of hyphens between the 'for' and 'against'
# columns, and this column has no header.
# To account for this, I set header=0 to discard the top row from the file,
# thus allowing me to use my own set of column headers.
# I specify the headers in the 'names' parameter.
table = pd.read_table('football.dat', delim_whitespace=True, header=0, \
names=['number', 'name', 'p', 'w', 'l', 'd', 'for', 'dash', 'against', 'pts'])
# convert the 'for' and 'against' series into numpy float arrays
pts_agnst, pts_for = np.float_(table['against']), np.float_(table['for'])
# create a numpy float array that contains the absolute values of the differences between 'against' and 'for'
spread = abs(pts_agnst - pts_for)
# Now, get the array index of the smallest spread value that is not == nan
# Ignoring nan is necessary to account for the row of hyphens. Another option would have been to
# skip this row in pd.read_table, but the approach I have used is more general.
# numpy.nanargmin does exactly this, finding the index of a float array for the minimum value in
# that array which is not nan
answer_idx = np.nanargmin(spread)
# Print the value in the name column for the row with the smallest spread
print(table['name'][answer_idx])
def pre_sev(self,
earth_radius=6378.0,
vertical_resolution=0.1,
cloud_top_height=1.0,
cloud_bottom_height=0.5):
self.lay = {}
self.lev = {}
f = SD(self.fname_h4, SDC.READ)
# lon lat
lon0 = f.select('Longitude')
lat0 = f.select('Latitude')
cot0 = f.select('Cloud_Optical_Thickness_16')
cer0 = f.select('Cloud_Effective_Radius_16')
cot_pcl0 = f.select('Cloud_Optical_Thickness_16_PCL')
cer_pcl0 = f.select('Cloud_Effective_Radius_16_PCL')
cth0 = f.select('Cloud_Top_Height')
if 'actual_range' in lon0.attributes().keys():
lon_range = lon0.attributes()['actual_range']
lat_range = lat0.attributes()['actual_range']
else:
lon_range = [-180.0, 180.0]
lat_range = [-90.0, 90.0]
lon = lon0[:]
lat = lat0[:]
cot = np.float_(cot0[:])
cer = np.float_(cer0[:])
cot_pcl = np.float_(cot_pcl0[:])
cer_pcl = np.float_(cer_pcl0[:])
cth = np.float_(cth0[:])
logic = (lon >= lon_range[0]) & (lon <= lon_range[1]) & (
lat >= lat_range[0]) & (lat <= lat_range[1])
lon = lon[logic]
lat = lat[logic]
cot = cot[logic]
cer = cer[logic]
cot_pcl = cot_pcl[logic]
cer_pcl = cer_pcl[logic]
cth = cth[logic]
if self.extent is not None:
logic = (lon >= self.extent[0]) & (lon <= self.extent[1]) & (
lat >= self.extent[2]) & (lat <= self.extent[3])
lon = lon[logic]
lat = lat[logic]
cot = cot[logic]
cer = cer[logic]
cot_pcl = cot_pcl[logic]
cer_pcl = cer_pcl[logic]
cth = cth[logic]
xy = (self.extent[1] - self.extent[0]) * (self.extent[3] -
self.extent[2])
N0 = np.sqrt(lon.size / xy)
Nx = int(N0 * (self.extent[1] - self.extent[0]))
if Nx % 2 == 1:
Nx += 1
Ny = int(N0 * (self.extent[3] - self.extent[2]))
if Ny % 2 == 1:
Ny += 1
lon_1d0 = np.linspace(self.extent[0], self.extent[1], Nx + 1)
lat_1d0 = np.linspace(self.extent[2], self.extent[3], Ny + 1)
lon_1d = (lon_1d0[1:] + lon_1d0[:-1]) / 2.0
lat_1d = (lat_1d0[1:] + lat_1d0[:-1]) / 2.0
dx = (lon_1d[1] - lon_1d[0]) / 180.0 * (np.pi * earth_radius)
dy = (lat_1d[1] - lat_1d[0]) / 180.0 * (np.pi * earth_radius)
x_1d = (lon_1d - lon_1d[0]) * dx
y_1d = (lat_1d - lat_1d[0]) * dy
# lon, lat
lat_2d, lon_2d = np.meshgrid(lat_1d, lon_1d)
# lon_2d, lat_2d = np.meshgrid(lon_1d, lat_1d)
# cot
cot_range = cot0.attributes()['valid_range']
cer_range = cer0.attributes()['valid_range']
cot_pcl_range = cot_pcl0.attributes()['valid_range']
cer_pcl_range = cer_pcl0.attributes()['valid_range']
cth_range = cth0.attributes()['valid_range']
# +
# create cot_all/cer_all that contains both cot/cer and cot_pcl/cer_pcl
cot_all = np.zeros(cot.size, dtype=np.float64)
cer_all = np.zeros(cer.size, dtype=np.float64)
cth_all = np.zeros(cth.size, dtype=np.float64)
cth_all[...] = np.nan
logic = (cot >= cot_range[0]) & (cot <= cot_range[1]) & (
cer >= cer_range[0]) & (cer <= cer_range[1]) & (
cth >= cth_range[0]) & (cth <= cth_range[1])
cot_all[logic] = cot[logic] * cot0.attributes(
)['scale_factor'] + cot0.attributes()['add_offset']
cer_all[logic] = cer[logic] * cer0.attributes(
)['scale_factor'] + cer0.attributes()['add_offset']
cth_all[logic] = cth[logic] * cth0.attributes(
)['scale_factor'] + cth0.attributes()['add_offset']
logic_pcl = np.logical_not(logic) & (cot_pcl >= cot_pcl_range[0]) & (
cot_pcl <= cot_pcl_range[1]) & (cer_pcl >= cer_pcl_range[0]) & (
cer_pcl <= cer_pcl_range[1]) & (cth >= cth_range[0]) & (
cth <= cth_range[1])
cot_all[logic_pcl] = cot_pcl[logic_pcl] * cot_pcl0.attributes(
)['scale_factor'] + cot_pcl0.attributes()['add_offset']
cer_all[logic_pcl] = cer_pcl[logic_pcl] * cer_pcl0.attributes(
)['scale_factor'] + cer_pcl0.attributes()['add_offset']
cth_all[logic_pcl] = cth[logic_pcl] * cth0.attributes(
)['scale_factor'] + cth0.attributes()['add_offset']
cth_all /= 1000.0
logic_all = logic | logic_pcl
# -
cot = cot_all
cer = cer_all
cth = cth_all
cot[np.logical_not(logic_all)] = 0.0
cer[np.logical_not(logic_all)] = 1.0
cth[np.logical_not(logic_all)] = np.nan
points = np.transpose(np.vstack((lon, lat)))
cot_2d = interpolate.griddata(points,
cot, (lon_2d, lat_2d),
method='nearest')
cer_2d = interpolate.griddata(points,
cer, (lon_2d, lat_2d),
method='nearest')
cth_2d = interpolate.griddata(points,
cth, (lon_2d, lat_2d),
method='nearest')
f.end()
# self.atm = atm_atmmod(np.arange(int(np.nanmax(cth_2d))+2))
self.atm = atm_atmmod(
levels=np.arange(cloud_bottom_height, cloud_top_height +
vertical_resolution, vertical_resolution))
self.lay['x'] = {'data': x_1d, 'name': 'X', 'units': 'km'}
self.lay['y'] = {'data': y_1d, 'name': 'Y', 'units': 'km'}
self.lay['nx'] = {'data': Nx, 'name': 'Nx', 'units': 'N/A'}
self.lay['ny'] = {'data': Ny, 'name': 'Ny', 'units': 'N/A'}
self.lay['dx'] = {'data': dx, 'name': 'dx', 'units': 'km'}
self.lay['dy'] = {'data': dy, 'name': 'dy', 'units': 'km'}
self.lay['altitude'] = copy.deepcopy(self.atm.lay['altitude'])
self.lay['cot'] = {
'data': cot_2d,
'name': 'Cloud optical thickness',
'units': 'N/A'
}
self.lay['cer'] = {
'data': cer_2d,
'name': 'Cloud effective radius',
'units': 'micron'
}
self.lay['cth'] = {
'data': cth_2d,
'name': 'Cloud top height',
'units': 'km'
}
self.lay['lon'] = {
'data': lon_2d,
'name': 'Longitude',
'units': 'degree'
}
self.lay['lat'] = {
'data': lat_2d,
'name': 'Latitude',
'units': 'degree'
}
# temperature 3d
t_1d = self.atm.lay['temperature']['data']
Nz = t_1d.size
t_3d = np.empty((Nx, Ny, Nz), dtype=t_1d.dtype)
t_3d[...] = t_1d[None, None, :]
self.lay['temperature'] = {
'data': t_3d,
'name': 'Temperature',
'units': 'K'
}
# extinction 3d
ext_3d = np.zeros((Nx, Ny, Nz), dtype=np.float64)
# alt = self.atm.lay['altitude']['data']
# for i in range(Nx):
# for j in range(Ny):
# cld_top = cth_2d[i, j]
# if not np.isnan(cld_top):
# lwp = 5.0/9.0 * 1.0 * cot_2d[i, j] * cer_2d[i, j] / 10.0
# ext0 = 0.75 * 2.0 * lwp / cer_2d[i, j] / 100.0
# index = np.argmin(np.abs(cld_top-alt))
# ext_3d[i, j, index] = ext0
for i in range(Nx):
for j in range(Ny):
ext0 = cal_ext(cot_2d[i, j], cer_2d[i, j])
ext_3d[i, j, :] = ext0 / (
self.atm.lay['thickness']['data'].sum() * 1000.0)
self.lay['extinction'] = {
'data': ext_3d,
'name': 'Extinction coefficients'
}
self.Nx = Nx
self.Ny = Ny
self.Nz = Nz
def read_potential(self, fileobj):
"""Reads a LAMMPS EAM file in alloy or adp format
and creates the interpolation functions from the data
"""
if isinstance(fileobj, str):
f = open(fileobj)
if self.form is None:
self.set_form(fileobj)
else:
f = fileobj
def lines_to_list(lines):
"""Make the data one long line so as not to care how its formatted
"""
data = []
for line in lines:
data.extend(line.split())
return data
lines = f.readlines()
if self.form == 'eam': # single element eam file (aka funcfl)
self.header = lines[:1]
data = lines_to_list(lines[1:])
# eam form is just like an alloy form for one element
self.Nelements = 1
self.Z = np.array([data[0]], dtype=int)
self.mass = np.array([data[1]])
self.a = np.array([data[2]])
self.lattice = [data[3]]
self.nrho = int(data[4])
self.drho = float(data[5])
self.nr = int(data[6])
self.dr = float(data[7])
self.cutoff = float(data[8])
n = 9 + self.nrho
self.embedded_data = np.array([np.float_(data[9:n])])
self.rphi_data = np.zeros(
[self.Nelements, self.Nelements, self.nr])
effective_charge = np.float_(data[n:n + self.nr])
# convert effective charges to rphi according to
# http://lammps.sandia.gov/doc/pair_eam.html
self.rphi_data[0, 0] = Bohr * Hartree * (effective_charge**2)
self.density_data = np.array(
[np.float_(data[n + self.nr:n + 2 * self.nr])])
elif self.form in ['alloy', 'adq']:
self.header = lines[:3]
i = 3
data = lines_to_list(lines[i:])
self.Nelements = int(data[0])
d = 1
self.elements = data[d:d + self.Nelements]
d += self.Nelements
self.nrho = int(data[d])
self.drho = float(data[d + 1])
self.nr = int(data[d + 2])
self.dr = float(data[d + 3])
self.cutoff = float(data[d + 4])
self.embedded_data = np.zeros([self.Nelements, self.nrho])
self.density_data = np.zeros([self.Nelements, self.nr])
self.Z = np.zeros([self.Nelements], dtype=int)
self.mass = np.zeros([self.Nelements])
self.a = np.zeros([self.Nelements])
self.lattice = []
d += 5
# reads in the part of the eam file for each element
for elem in range(self.Nelements):
self.Z[elem] = int(data[d])
self.mass[elem] = float(data[d + 1])
self.a[elem] = float(data[d + 2])
self.lattice.append(data[d + 3])
d += 4
self.embedded_data[elem] = np.float_(data[d:(d + self.nrho)])
d += self.nrho
self.density_data[elem] = np.float_(data[d:(d + self.nr)])
d += self.nr
# reads in the r*phi data for each interaction between elements
self.rphi_data = np.zeros(
[self.Nelements, self.Nelements, self.nr])
for i in range(self.Nelements):
for j in range(i + 1):
self.rphi_data[j, i] = np.float_(data[d:(d + self.nr)])
d += self.nr
elif self.form == 'fs':
self.header = lines[:3]
i = 3
data = lines_to_list(lines[i:])
self.Nelements = int(data[0])
d = 1
self.elements = data[d:d + self.Nelements]
d += self.Nelements
self.nrho = int(data[d])
self.drho = float(data[d + 1])
self.nr = int(data[d + 2])
self.dr = float(data[d + 3])
self.cutoff = float(data[d + 4])
self.embedded_data = np.zeros([self.Nelements, self.nrho])
self.density_data = np.zeros(
[self.Nelements, self.Nelements, self.nr])
self.Z = np.zeros([self.Nelements], dtype=int)
self.mass = np.zeros([self.Nelements])
self.a = np.zeros([self.Nelements])
self.lattice = []
d += 5
# reads in the part of the eam file for each element
for elem in range(self.Nelements):
self.Z[elem] = int(data[d])
self.mass[elem] = float(data[d + 1])
self.a[elem] = float(data[d + 2])
self.lattice.append(data[d + 3])
d += 4
self.embedded_data[elem] = np.float_(data[d:(d + self.nrho)])
d += self.nrho
self.density_data[elem, :, :] = np.float_(
data[d:(d + self.nr * self.Nelements)]).reshape(
[self.Nelements, self.nr])
d += self.nr * self.Nelements
# reads in the r*phi data for each interaction between elements
self.rphi_data = np.zeros(
[self.Nelements, self.Nelements, self.nr])
for i in range(self.Nelements):
for j in range(i + 1):
self.rphi_data[j, i] = np.float_(data[d:(d + self.nr)])
d += self.nr
self.r = np.arange(0, self.nr) * self.dr
self.rho = np.arange(0, self.nrho) * self.drho
# choose the set_splines method according to the type
if self.form == 'fs':
self.set_fs_splines()
else:
self.set_splines()
if (self.form == 'adp'):
self.read_adp_data(data, d)
self.set_adp_splines()
h.write_map(dir_out + '/sin_r1_' + str(i + 1) + '_tmp.fits', sin_r1)
h.write_map(dir_out + '/cos_r2_' + str(i + 1) + '_tmp.fits', cos_r2)
h.write_map(dir_out + '/sin_r2_' + str(i + 1) + '_tmp.fits', sin_r2)
h.write_map(dir_out + '/cos_r4_' + str(i + 1) + '_tmp.fits', cos_r4)
h.write_map(dir_out + '/sin_r4_' + str(i + 1) + '_tmp.fits', sin_r4)
if (i == ydays - 1):
h.write_map(dir_out + '/nhits_' + str(i + 1) + '_tmp.fits', nhits)
h.write_map(dir_out + '/cos_r1_' + str(i + 1) + '_tmp.fits', cos_r1)
h.write_map(dir_out + '/sin_r1_' + str(i + 1) + '_tmp.fits', sin_r1)
h.write_map(dir_out + '/cos_r2_' + str(i + 1) + '_tmp.fits', cos_r2)
h.write_map(dir_out + '/sin_r2_' + str(i + 1) + '_tmp.fits', sin_r2)
h.write_map(dir_out + '/cos_r4_' + str(i + 1) + '_tmp.fits', cos_r4)
h.write_map(dir_out + '/sin_r4_' + str(i + 1) + '_tmp.fits', sin_r4)
r1 = np.sqrt((cos_r1 / np.float_(nhits))**2. + (sin_r1 / np.float_(nhits))**2.)
r2 = np.sqrt((cos_r2 / np.float_(nhits))**2. + (sin_r2 / np.float_(nhits))**2.)
r4 = np.sqrt((cos_r4 / np.float_(nhits))**2. + (sin_r4 / np.float_(nhits))**2.)
filename_out = dir_out + '/' + filename + '_nhits'
h.write_map(filename_out + '.fits', nhits)
h.mollview(nhits, title='Nobs, ' + filename_out, rot=[0., 0.])
h.graticule(dpar=10, dmer=10, coord='E')
py.savefig(filename_out + '.ps')
filename_out = dir_out + '/' + filename + '_cos_r1'
h.write_map(filename_out + '.fits', cos_r1)
h.mollview(cos_r1, title='cos_r1, ' + filename_out, rot=[0., 0.])
h.graticule(dpar=10, dmer=10, coord='E')
py.savefig(filename_out + '.ps')
def Download_data(output_folder_Tot, Date, latlim, lonlim):
polygon = [[[lonlim[0], latlim[0]], [lonlim[0], latlim[1]],
[lonlim[1], latlim[1]], [lonlim[1], latlim[0]],
[lonlim[0], latlim[0]]]]
url = 'https://io.apps.fao.org/gismgr/api/v1/query/'
header = {
"Content-type": "application/json;charset=UTF-8",
"Accept": "application/json"
}
# Create request
payload = {
"type": "CropRaster",
"params": {
"properties": {
"outputFileName": "LC_WAPOR_%s.01.01.tif" % Date.year,
"cutline": True,
"tiled": True,
"compressed": True,
"overviews": True
},
"cube": {
"code": "L1_LCC_A",
"workspaceCode": "WAPOR",
"language": "en"
},
"dimensions": [{
"code":
"YEAR",
"values":
["[%s-01-01,%s-01-01)" % (int(Date.year), int(Date.year) + 1)]
}],
"measures": ["LCC"],
"shape": {
"type": "Polygon",
"coordinates": polygon
}
}
}
# Download the data
response = requests.post(url, data=json.dumps(payload), headers=header)
response.raise_for_status()
response_json = response.json()
result = response_json['response']
job_url = result['links'][0]['href']
for tries in range(20):
file_name_temp = os.path.join(output_folder_Tot,
"LC_WAPOR_%s.01.01_temp.tif" % Date.year)
if not os.path.exists(file_name_temp):
time.sleep(5)
job_response = requests.get(job_url, headers=header)
if job_response.status_code == 200:
try:
job_result = job_response.json(
)['response']['output']['downloadUrl']
urllib.request.urlretrieve(job_result, file_name_temp)
dest = gdal.Open(file_name_temp)
geo = dest.GetGeoTransform()
proj = dest.GetProjection()
Array = dest.GetRasterBand(1).ReadAsArray()
Array = np.float_(Array)
except:
pass
return (Array, geo, proj, file_name_temp)
import os.path as ptt
import my
from my import mycall
def sycall(comand):
from subprocess import call
line = comand.split(" ")
fcomand = []
fcomand.extend(line)
call(fcomand)
def wfits(name, data, hdr):
if ptt.exists(name) == False:
wfit(name, data, hdr)
else:
sycall("rm " + name)
wfit(name, data, hdr)
sys.argv = filter(None, sys.argv)
if len(sys.argv) < 2:
print "USE: write_img_header.py FILE.FITS HEADER VALUE"
sys.exit(0)
file = sys.argv[1]
header = sys.argv[2]
value = float_(sys.argv[3])
[pdl, h] = gdata(file, 0, header=True)
h[header] = value
wfits(file, pdl, h)
