def get_coordinates(*args):
"""
Returns the array of coordinates of the elements of the other arrays :
either an array of tuples (theta, phi) or of tuples (rho, phi, z) - either in (degrees, degrees) or in (meters, degrees, meters).
"""
_ifar = _get_ifar(*args)
n_theta=_get_n_theta(*args)
n_phi=_get_n_phi(*args)
theta_start = _get_theta_start(*args)
delta_theta = _get_delta_theta(*args)
phi_start = _get_phi_start(*args)
delta_phi = _get_delta_phi(*args)
l=[]
if _ifar == 1 :
rho = _PyNEC.nec_radiation_pattern_get_range(*args)
for i in range(n_phi) :
for j in range(n_theta) :
l.append((rho, phi_start+i*delta_phi,theta_start+j*delta_theta))
ar = numarray.array(l);
ar = numarray.reshape(ar, (n_phi,n_theta,3))
else :
for i in range(n_phi) :
for j in range(n_theta) :
l.append((theta_start+j*delta_theta, phi_start+i*delta_phi))
ar = numarray.array(l);
ar = numarray.reshape(ar, (n_phi,n_theta,2))
return ar
def mask(clus_id, line_s):
imagefile = c.imagefile
sex_cata = c.sex_cata
threshold = c.threshold
thresh_area = c.thresh_area
size = c.size
mask_reg = c.mask_reg
x = n.reshape(n.arange(size*size),(size,size)) % size
x = x.astype(n.Float32)
y = n.reshape(n.arange(size*size),(size,size)) / size
y = y.astype(n.Float32)
values = line_s.split()
mask_file = 'mask_' + str(imagefile)[:6] + '_' + str(clus_id) + '.fits'
xcntr_o = float(values[1]) #x center of the object
ycntr_o = float(values[2]) #y center of the object
xcntr = size / 2.0 + 1.0 + xcntr_o - int(xcntr_o)
ycntr = size / 2.0 + 1.0 + ycntr_o - int(ycntr_o)
mag = float(values[7]) #Magnitude
radius = float(values[9]) #Half light radius
mag_zero = c.mag_zero #magnitude zero point
sky = float(values[10]) #sky
pos_ang = float(values[11]) - 90.0 #position angle
axis_rat = 1.0 / float(values[12]) #axis ration b/a
major_axis = float(values[14]) #major axis of the object
z = n.zeros((size,size))
for line_j in open(sex_cata,'r'):
try:
values = line_j.split()
xcntr_n = float(values[1]) #x center of the neighbour
ycntr_n = float(values[2]) #y center of the neighbour
mag = float(values[7]) #Magnitude
radius = float(values[9]) #Half light radius
sky = float(values[10]) #sky
pos_ang = float(values[11]) - 90.0 #position angle
axis_rat = 1.0/float(values[12]) #axis ration b/a
area = float(values[13])
maj_axis = float(values[14])#major axis of neighbour
if(abs(xcntr_n - xcntr_o) < size/2.0 and abs(ycntr_n - ycntr_o) \
< size/2.0 and area < thresh_area):
if(abs(xcntr_n - xcntr_o) >= major_axis * threshold or \
abs(ycntr_n - ycntr_o) >= major_axis * threshold):
if((xcntr_o - xcntr_n) < 0):
xn = xcntr + abs(xcntr_n - xcntr_o)
if((ycntr_o - ycntr_n) < 0):
yn = ycntr + abs(ycntr_n - ycntr_o)
if((xcntr_o - xcntr_n) > 0):
xn = xcntr - (xcntr_o - xcntr_n)
if((ycntr_o - ycntr_n) > 0):
yn = ycntr - (ycntr_o - ycntr_n)
tx = x - xn + 0.5
ty = y - yn + 0.5
R = n.sqrt(tx**2.0 + ty**2.0)
z[n.where(R<=mask_reg*maj_axis)] = 1
except:
pass
hdu = pyfits.PrimaryHDU(z.astype(n.Float32))
hdu.writeto(mask_file)
def _reshape(arg0,arg1):
"""
Changes the shape of the array arg0 to (n_phi, n_theta) to balance the effect of the typemaps which return flat arrays.
"""
n_theta=_get_n_theta(arg1)
n_phi=_get_n_phi(arg1)
return numarray.reshape(arg0,(n_phi,n_theta))
def read_data(filename, hdu = -1, verbose=0):
#####################################################################
# Reads and return a NumPy array containing the data in extension #
# hdu. #
#####################################################################
if verbose: print "Opening ",filename,
p = pcfitsio.fits_open_file(filename,0)
if verbose: print ". Done."
if hdu!=-1:
pcfitsio.fits_movabs_hdu(p,hdu)
if verbose: print "Reading image size.",
n = pcfitsio.fits_read_keys_lng(p,"NAXIS",1,10)
if verbose: print n,"Done."
if n==[] or n==[0]:
pcfitsio.fits_close_file(p)
raise KeyError," Could not read NAXIS values"
if verbose: print "Reading data.",
data = 0
l = []
for i in range(len(n)):
l.append(n[len(n)-i-1])
data = pcfitsio.fits_read_img(p,1,numarray.multiply.reduce(l) ,0.)[0]
if verbose: print "Data read ok.",numarray.shape(data),numarray.multiply.reduce(l)
if verbose: print "Reshape ",l
data=numarray.reshape(data,l)
if verbose: print "Reshape ",l," ok."
if verbose: print "Read",n,"=",numarray.multiply.reduce(n),"elements.",
if verbose: print "Done."
if verbose: print "Closing",filename,
pcfitsio.fits_close_file(p)
if verbose: print ".Done."
return data
def ConvertMosaic(fits, outPath, percent=5, levels=[None, None]):
print "\nConversion from FITS to JPG"
print "Infile: ", fits
if (not levels[0]) and (not levels[1]):
# Compute the cut levels
fimg = pyfits.open(fits)
data = fimg[0].data
nElem = data.nelements()
lData = numarray.reshape(data, nElem)
if nElem > 2e6:
# Sample a million pixels from within the image
lData = lData[int(nElem/2-5e5):int(nElem/2+5e5)]
nElem = lData.nelements()
lData.sort()
# Get the 2.5 percentiles (default: percent = 5)
percent /= 200.
if not levels[0]:
minVal = lData[int(nElem*percent)]
else:
minVal = levels[0]
if not levels[1]:
maxVal = lData[int(nElem*(1-percent))-1]
else:
maxVal = levels[1]
fimg.close()
else:
minVal = levels[0]
maxVal = levels[1]
psFile = tempfile.mktemp(".ps")
wipfileName = tempfile.mktemp(".wip")
wipfile = open(wipfileName, "w")
wipfile.write("device %s/ps\n" % (psFile, ))
wipfile.write("image %s\n" % (fits,))
wipfile.write("winadj 0 nx 0 ny\n")
wipfile.write("halftone %f %f\n" % (minVal, maxVal))
wipfile.close()
cmd = "wip -d /ps < %s >/dev/null " % (wipfileName, )
os.system(cmd)
os.remove(wipfileName)
jpgFile = os.path.join(outPath, "images", os.path.split(fits)[1][:-5]+".jpg")
cmd = "convert -rotate 90 %s %s" % (psFile, jpgFile)
os.system(cmd)
print "Outfile: ", jpgFile
print
if not os.path.isfile(jpgFile):
return "Creation of %s failed" % (jpgFile,), None
os.remove(psFile)
return "ok", jpgFile
def _get_mag(arg0):
"""
Returns the array of receiving gain not yet normalized.
"""
n_theta = _get_n_theta(arg0)
n_phi = _get_n_phi(arg0)
ar = numarray.reshape(_PyNEC.nec_norm_rx_pattern_get_mag(arg0),(n_theta, n_phi))
ar.transpose()
return ar
def __call__(self, lon, lat, inverse=False):
"""
Calling a Proj class instance with the arguments lon, lat will
convert lon/lat (in degrees) to x/y native map projection
coordinates (in meters). If optional keyword 'inverse' is
True (default is False), the inverse transformation from x/y
to lon/lat is performed.
For cylindrical equidistant projection ('cyl'), this
does nothing (i.e. x,y == lon,lat).
For mercator projection ('merc'), x == lon, but y has units
of meters.
lon,lat can be either scalar floats or N arrays.
"""
if self.projection == 'cyl': # for cyl x,y == lon,lat
return lon, lat
# if inputs are numarray arrays, get shape and typecode.
try:
shapein = lon.shape
lontypein = lon.typecode()
lattypein = lat.typecode()
except:
shapein = False
# make sure inputs have same shape.
if shapein and lat.shape != shapein:
raise ValueError, 'lon, lat must be scalars or numarrays with the same shape'
if shapein:
x = N.ravel(lon).tolist()
y = N.ravel(lat).tolist()
if inverse:
outx, outy = self._inv(x, y)
else:
outx, outy = self._fwd(x, y)
outx = N.reshape(N.array(outx, lontypein), shapein)
outy = N.reshape(N.array(outy, lattypein), shapein)
else:
if inverse:
outx, outy = self._inv(lon, lat)
else:
outx, outy = self._fwd(lon, lat)
return outx, outy
def __call__(self,lon,lat,inverse=False):
"""
Calling a Proj class instance with the arguments lon, lat will
convert lon/lat (in degrees) to x/y native map projection
coordinates (in meters). If optional keyword 'inverse' is
True (default is False), the inverse transformation from x/y
to lon/lat is performed.
For cylindrical equidistant projection ('cyl'), this
does nothing (i.e. x,y == lon,lat).
For mercator projection ('merc'), x == lon, but y has units
of meters.
lon,lat can be either scalar floats or N arrays.
"""
if self.projection == 'cyl': # for cyl x,y == lon,lat
return lon,lat
# if inputs are numarray arrays, get shape and typecode.
try:
shapein = lon.shape
lontypein = lon.typecode()
lattypein = lat.typecode()
except:
shapein = False
# make sure inputs have same shape.
if shapein and lat.shape != shapein:
raise ValueError, 'lon, lat must be scalars or numarrays with the same shape'
if shapein:
x = N.ravel(lon).tolist()
y = N.ravel(lat).tolist()
if inverse:
outx, outy = self._inv(x,y)
else:
outx, outy = self._fwd(x,y)
outx = N.reshape(N.array(outx,lontypein),shapein)
outy = N.reshape(N.array(outy,lattypein),shapein)
else:
if inverse:
outx,outy = self._inv(lon,lat)
else:
outx,outy = self._fwd(lon,lat)
return outx,outy
def get_coordinates(*args):
"""
Returns the array of coordinates of the elements of the other arrays :
an array of tuples (theta, phi) - in (degrees, degrees).
"""
n_theta=_get_n_theta(*args)
n_phi=_get_n_phi(*args)
theta_start = _get_theta_start(*args)
delta_theta = _get_delta_theta(*args)
phi_start = _get_phi_start(*args)
delta_phi = _get_delta_phi(*args)
l=[]
for i in range(n_phi) :
for j in range(n_theta) :
l.append((theta_start+j*delta_theta, phi_start+i*delta_phi))
ar = numarray.array(l);
ar = numarray.reshape(ar, (n_phi,n_theta,2))
return ar
def process(self, X, V, C):
BifPoint.process(self, X, V, C)
J_coords = C.CorrFunc.sysfunc.jac(X, C.coords)
B = C.CorrFunc.sysfunc.hess(X, C.coords, C.coords)
W, VL, VR = linalg.eig(J_coords, left=1, right=1)
q = C.CorrFunc.testfunc.data.C/linalg.norm(C.CorrFunc.testfunc.data.C)
p = C.CorrFunc.testfunc.data.B/matrixmultiply(transpose(C.CorrFunc.testfunc.data.B),q)
self.found[-1].eigs = W
a = 0.5*matrixmultiply(transpose(p), reshape([bilinearform(B[i,:,:], q, q) \
for i in range(B.shape[0])],(B.shape[0],1)))[0][0]
if C.verbosity >= 2:
print '\nChecking...'
print ' |a| = %f' % a
print '\n'
self.info(C, -1)
return True
def __call__(self,lon,lat,inverse=False):
"""
Calling a Proj class instance with the arguments lon, lat will
convert lon/lat (in degrees) to x/y native map projection
coordinates (in meters). If optional keyword 'inverse' is
True (default is False), the inverse transformation from x/y
to lon/lat is performed.
Example usage:
>>> from proj import Proj
>>> params = {}
>>> params['proj'] = 'lcc' # lambert conformal conic
>>> params['lat_1'] = 30. # standard parallel 1
>>> params['lat_2'] = 50. # standard parallel 2
>>> params['lon_0'] = -96 # central meridian
>>> map = Proj(params)
>>> x,y = map(-107,40) # find x,y coords of lon=-107,lat=40
>>> print x,y
-92272.1004461 477477.890988
lon,lat can be either scalar floats or numarray arrays.
"""
npts = 1
# if inputs are numarray arrays, get shape and total number of pts.
try:
shapein = lon.shape
npts = reduce(lambda x, y: x*y, shapein)
lontypein = lon.typecode()
lattypein = lat.typecode()
except:
shapein = False
# make sure inputs have same shape.
if shapein and lat.shape != shapein:
raise ValueError, 'lon, lat must be scalars or numarray arrays with the same shape'
# make a rank 1 array with x/y (or lon/lat) pairs.
xy = numarray.zeros(2*npts,'d')
xy[::2] = numarray.ravel(lon)
xy[1::2] = numarray.ravel(lat)
# convert degrees to radians, meters to cm.
if not inverse:
xy = _dg2rad*xy
else:
xy = 10.*xy
# write input data to binary file.
xyout = array.array('d')
xyout.fromlist(xy.tolist())
# set up cmd string, scale factor for output.
if inverse:
cmd = self.projcmd+' -I'
scale = _rad2dg
else:
cmd = self.projcmd
scale = 0.1
# use pipes for input and output if amount of
# data less than default buffer size (usually 8192).
# otherwise use pipe for input, tempfile for output
if 2*npts*8 > 8192:
fd, fn=tempfile.mkstemp(); os.close(fd); stdout=open(fn,'rb')
cmd = cmd+' > '+fn
stdin=os.popen(cmd,'w')
xyout.tofile(stdin)
stdin.close()
outxy = scale*numarray.fromstring(stdout.read(),'d')
stdout.close()
os.remove(fn)
else:
stdin,stdout=os.popen2(cmd,mode='b')
xyout.tofile(stdin)
stdin.close()
outxy = scale*numarray.fromstring(stdout.read(),'d')
stdout.close()
# return numarray arrays or scalars, depending on type of input.
if shapein:
outx = numarray.reshape(outxy[::2],shapein).astype(lontypein)
outy = numarray.reshape(outxy[1::2],shapein).astype(lattypein)
else:
outx = outxy[0]; outy = outxy[1]
return outx,outy
def mask(clus_id, line_s):
imagefile = c.imagefile
sex_cata = c.sex_cata
clus_cata = c.out_cata
threshold = c.threshold
thresh_area = c.thresh_area
size = c.size
mask_reg = c.mask_reg
x = n.reshape(n.arange(size*size),(size,size)) % size
x = x.astype(n.Float32)
y = n.reshape(n.arange(size*size),(size,size)) / size
y = y.astype(n.Float32)
values = line_s.split()
mask_file = 'ell_mask_' + str(imagefile)[:6] + '_' + str(clus_id) + '.fits'
xcntr_o = float(values[1]) #x center of the object
ycntr_o = float(values[2]) #y center of the object
xcntr = size / 2.0 + 1.0 + xcntr_o - int(xcntr_o)
ycntr = size / 2.0 + 1.0 + ycntr_o - int(ycntr_o)
mag = float(values[7]) #Magnitude
radius = float(values[9]) #Half light radius
mag_zero = c.mag_zero #magnitude zero point
sky = float(values[10]) #sky
pos_ang = float(values[11]) - 90.0 #position angle
axis_rat = 1.0 / float(values[12]) #axis ration b/a
major_axis = float(values[14]) #major axis of the object
z = n.zeros((size,size))
for line_j in open(sex_cata,'r'):
try:
values = line_j.split()
xcntr_n = float(values[1]) #x center of the neighbour
ycntr_n = float(values[2]) #y center of the neighbour
mag = float(values[7]) #Magnitude
radius = float(values[9]) #Half light radius
sky = float(values[10]) #sky
pos_ang = float(values[11]) #position angle
axis_rat = 1.0/float(values[12]) #axis ration b/a
si = n.sin(pos_ang * n.pi / 180.0)
co = n.cos(pos_ang * n.pi / 180.0)
area = float(values[13])
maj_axis = float(values[14])#major axis of neighbour
eg = 1.0 - axis_rat
one_minus_eg_sq = (1.0-eg)**2.0
if(abs(xcntr_n - xcntr_o) < size/2.0 and \
abs(ycntr_n - ycntr_o) < size/2.0 and \
xcntr_n != xcntr_o and ycntr_n != ycntr_o):
if((xcntr_o - xcntr_n) < 0):
xn = xcntr + abs(xcntr_n - xcntr_o)
if((ycntr_o - ycntr_n) < 0):
yn = ycntr + abs(ycntr_n - ycntr_o)
if((xcntr_o - xcntr_n) > 0):
xn = xcntr - (xcntr_o -xcntr_n)
if((ycntr_o - ycntr_n) > 0):
yn = ycntr - (ycntr_o -ycntr_n)
tx = (x - xn + 0.5) * co + (y - yn + 0.5) * si
ty = (xn - 0.5 -x) * si + (y - yn + 0.5) * co
R = n.sqrt(tx**2.0 + ty**2.0 / one_minus_eg_sq)
z[n.where(R<=mask_reg*maj_axis)] = 1
except:
i=1
hdu = pyfits.PrimaryHDU(z.astype(n.Float32))
hdu.writeto(mask_file)
def GetMedian(data):
n = data.nelements()
lData = numarray.reshape(data, n)
lData.sort()
return lData[int(n/2)]
def train(self, train_toks, **kwargs):
"""
Train a new C{ConditionalExponentialClassifier}, using the
given training samples. This
C{ConditionalExponentialClassifier} should encode the model
that maximizes entropy from all the models that are
emperically consistant with C{train_toks}.
@param kwargs: Keyword arguments.
- C{iterations}: The maximum number of times IIS should
iterate. If IIS converges before this number of
iterations, it may terminate. Default=C{20}.
(type=C{int})
- C{debug}: The debugging level. Higher values will cause
more verbose output. Default=C{0}. (type=C{int})
- C{classes}: The set of possible classes. If none is given,
then the set of all classes attested in the training data
will be used instead. (type=C{list} of (immutable)).
- C{accuracy_cutoff}: The accuracy value that indicates
convergence. If the accuracy becomes closer to one
than the specified value, then IIS will terminate. The
default value is None, which indicates that no accuracy
cutoff should be used. (type=C{float})
- C{delta_accuracy_cutoff}: The change in accuracy should be
taken to indicate convergence. If the accuracy changes by
less than this value in a single iteration, then IIS will
terminate. The default value is C{None}, which indicates
that no accuracy-change cutoff should be
used. (type=C{float})
- C{log_likelihood_cutoff}: specifies what log-likelihood
value should be taken to indicate convergence. If the
log-likelihod becomes closer to zero than the specified
value, then IIS will terminate. The default value is
C{None}, which indicates that no log-likelihood cutoff
should be used. (type=C{float})
- C{delta_log_likelihood_cutoff}: specifies what change in
log-likelihood should be taken to indicate convergence.
If the log-likelihood changes by less than this value in a
single iteration, then IIS will terminate. The default
value is C{None}, which indicates that no
log-likelihood-change cutoff should be used. (type=C{float})
"""
assert _chktype(1, train_toks, [Token], (Token, ))
# Process the keyword arguments.
iter = 20
debug = 0
classes = None
ll_cutoff = lldelta_cutoff = None
acc_cutoff = accdelta_cutoff = None
for (key, val) in kwargs.items():
if key in ('iterations', 'iter'): iter = val
elif key == 'debug': debug = val
elif key == 'classes': classes = val
elif key == 'log_likelihood_cutoff':
ll_cutoff = abs(val)
elif key == 'delta_log_likelihood_cutoff':
lldelta_cutoff = abs(val)
elif key == 'accuracy_cutoff':
acc_cutoff = abs(val)
elif key == 'delta_accuracy_cutoff':
accdelta_cutoff = abs(val)
else:
raise TypeError('Unknown keyword arg %s' % key)
if classes is None:
classes = attested_classes(train_toks)
self._classes = classes
# Find the classes, if necessary.
if classes is None:
classes = find_classes(train_toks)
# Find the length of the first token's feature vector.
if len(train_toks) == 0:
raise ValueError('Expected at least one training token')
vector0 = train_toks[0]['FEATURE_VECTOR']
self._feature_vector_len = len(vector0)
self._weight_vector_len = self._feature_vector_len * len(self._classes)
# Build the offsets dictionary. This maps from a class to the
# index in the weight vector where that class's weights begin.
self._offsets = dict([(cls, i * self._feature_vector_len)
for i, cls in enumerate(classes)])
# Find the frequency with which each feature occurs in the
# training data.
ffreq_emperical = self._ffreq_emperical(train_toks)
# Find the nf map, and related variables nfarray and nfident.
# nf is the sum of the features for a given labeled text.
# nfmap compresses this sparse set of values to a dense list.
# nfarray performs the reverse operation. nfident is
# nfarray multiplied by an identity matrix.
nfmap = self._nfmap(train_toks)
nfs = nfmap.items()
nfs.sort(lambda x, y: cmp(x[1], y[1]))
nfarray = numarray.array([nf for (nf, i) in nfs], 'd')
nftranspose = numarray.reshape(nfarray, (len(nfarray), 1))
# An array that is 1 whenever ffreq_emperical is zero. In
# other words, it is one for any feature that's not attested
# in the data. This is used to avoid division by zero.
unattested = numarray.zeros(self._weight_vector_len, 'd')
for i in range(len(unattested)):
if ffreq_emperical[i] == 0: unattested[i] = 1
# Build the classifier. Start with weight=1 for each feature,
# except for the unattested features. Start those out at
# zero, since we know that's the correct value.
weights = numarray.ones(self._weight_vector_len, 'd')
weights -= unattested
classifier = ConditionalExponentialClassifier(classes, weights)
if debug > 0: print ' ==> Training (%d iterations)' % iter
if debug > 2:
print
print ' Iteration Log Likelihood Accuracy'
print ' ---------------------------------------'
# Train for a fixed number of iterations.
for iternum in range(iter):
if debug > 2:
print(' %9d %14.5f %9.3f' %
(iternum,
classifier_log_likelihood(classifier, train_toks),
classifier_accuracy(classifier, train_toks)))
# Calculate the deltas for this iteration, using Newton's method.
deltas = self._deltas(train_toks, classifier, unattested,
ffreq_emperical, nfmap, nfarray, nftranspose)
# Use the deltas to update our weights.
weights = classifier.weights()
weights *= numarray.exp(deltas)
classifier.set_weights(weights)
# Check log-likelihood cutoffs.
if ll_cutoff is not None or lldelta_cutoff is not None:
ll = classifier_log_likelihood(classifier, train_toks)
if ll_cutoff is not None and ll > -ll_cutoff: break
if lldelta_cutoff is not None:
if (ll - ll_old) < lldelta_cutoff: break
ll_old = ll
# Check accuracy cutoffs.
if acc_cutoff is not None or accdelta_cutoff is not None:
acc = classifier_accuracy(classifier, train_toks)
if acc_cutoff is not None and acc < acc_cutoff: break
if accdelta_cutoff is not None:
if (acc_old - acc) < accdelta_cutoff: break
acc_old = acc
if debug > 2:
print(' %9d %14.5f %9.3f' %
(iternum + 1,
classifier_log_likelihood(classifier, train_toks),
classifier_accuracy(classifier, train_toks)))
print
# Return the classifier.
return classifier
def GetCutStdev(data):
n = data.nelements()
lData = numarray.reshape(data, n)
lData.sort()
cData = lData[int(n*0.025):int(n*0.975)]
return cData.stddev()
def interp(datain,lonsin,latsin,lonsout,latsout,checkbounds=False,mode='nearest',cval=0.0,order=3):
"""
dataout = interp(datain,lonsin,latsin,lonsout,latsout,mode='constant',cval=0.0,order=3)
interpolate data (datain) on a rectilinear lat/lon grid (with lons=lonsin
lats=latsin) to a grid with lons=lonsout, lats=latsout.
datain is a rank-2 array with 1st dimension corresponding to longitude,
2nd dimension latitude.
lonsin, latsin are rank-1 Numeric arrays containing longitudes and latitudes
of datain grid in increasing order (i.e. from Greenwich meridian eastward, and
South Pole northward)
lonsout, latsout are rank-2 Numeric arrays containing lons and lats out desired
output grid (typically a native map projection grid).
If checkbounds=True, values of lonsout and latsout are checked to see that
they lie within the range specified by lonsin and latsing. Default is
False, and values outside the borders are handled in the manner described
by the 'mode' parameter (default mode='nearest', which means the nearest
boundary value is used). See section 20.2 of the numarray docs for
information on the 'mode' keyword.
See numarray.nd_image.map_coordinates documentation for information on
the other optional keyword parameters. The order keyword can be 0
for nearest neighbor interpolation (nd_image only allows 1-6) - if
order=0 bounds checking is done even if checkbounds=False.
"""
# lonsin and latsin must be monotonically increasing.
if lonsin[-1]-lonsin[0] < 0 or latsin[-1]-latsin[0] < 0:
raise ValueError, 'lonsin and latsin must be increasing!'
# optionally, check that lonsout,latsout are
# within region defined by lonsin,latsin.
# (this check is always done if nearest neighbor
# interpolation (order=0) requested).
if checkbounds or order == 0:
if min(N.ravel(lonsout)) < min(lonsin) or \
max(N.ravel(lonsout)) > max(lonsin) or \
min(N.ravel(latsout)) < min(latsin) or \
max(N.ravel(latsout)) > max(latsin):
raise ValueError, 'latsout or lonsout outside range of latsin or lonsin'
# compute grid coordinates of output grid.
delon = lonsin[1:]-lonsin[0:-1]
delat = latsin[1:]-latsin[0:-1]
if max(delat)-min(delat) < 1.e-4 and max(delon)-min(delon) < 1.e-4:
# regular input grid.
xcoords = (len(lonsin)-1)*(lonsout-lonsin[0])/(lonsin[-1]-lonsin[0])
ycoords = (len(latsin)-1)*(latsout-latsin[0])/(latsin[-1]-latsin[0])
else:
# irregular (but still rectilinear) input grid.
lonsoutflat = N.ravel(lonsout)
latsoutflat = N.ravel(latsout)
ix = N.searchsorted(lonsin,lonsoutflat)-1
iy = N.searchsorted(latsin,latsoutflat)-1
xcoords = N.zeros(ix.shape,'f')
ycoords = N.zeros(iy.shape,'f')
for n,i in enumerate(ix):
if i < 0:
xcoords[n] = -1 # outside of range on lonsin (lower end)
elif i >= len(lonsin)-1:
xcoords[n] = len(lonsin) # outside range on upper end.
else:
xcoords[n] = float(i)+(lonsoutflat[n]-lonsin[i])/(lonsin[i+1]-lonsin[i])
xcoords = N.reshape(xcoords,lonsout.shape)
for m,j in enumerate(iy):
if j < 0:
ycoords[m] = -1 # outside of range of latsin (on lower end)
elif j >= len(latsin)-1:
ycoords[m] = len(latsin) # outside range on upper end
else:
ycoords[m] = float(j)+(latsoutflat[m]-latsin[j])/(latsin[j+1]-latsin[j])
ycoords = N.reshape(ycoords,latsout.shape)
coords = [ycoords,xcoords]
# interpolate to output grid using numarray.nd_image spline filter.
if order:
return nd_image.map_coordinates(datain,coords,mode=mode,cval=cval,order=order)
else:
# nearest neighbor interpolation if order=0.
# uses index arrays, so first convert to numarray.
datatmp = N.array(datain,datain.typecode())
xi = N.around(xcoords).astype('i')
yi = N.around(ycoords).astype('i')
return datatmp[yi,xi]
def interp(datain,
lonsin,
latsin,
lonsout,
latsout,
checkbounds=False,
mode='nearest',
cval=0.0,
order=3):
"""
dataout = interp(datain,lonsin,latsin,lonsout,latsout,mode='constant',cval=0.0,order=3)
interpolate data (datain) on a rectilinear lat/lon grid (with lons=lonsin
lats=latsin) to a grid with lons=lonsout, lats=latsout.
datain is a rank-2 array with 1st dimension corresponding to longitude,
2nd dimension latitude.
lonsin, latsin are rank-1 Numeric arrays containing longitudes and latitudes
of datain grid in increasing order (i.e. from Greenwich meridian eastward, and
South Pole northward)
lonsout, latsout are rank-2 Numeric arrays containing lons and lats out desired
output grid (typically a native map projection grid).
If checkbounds=True, values of lonsout and latsout are checked to see that
they lie within the range specified by lonsin and latsing. Default is
False, and values outside the borders are handled in the manner described
by the 'mode' parameter (default mode='nearest', which means the nearest
boundary value is used). See section 20.2 of the numarray docs for
information on the 'mode' keyword.
See numarray.nd_image.map_coordinates documentation for information on
the other optional keyword parameters. The order keyword can be 0
for nearest neighbor interpolation (nd_image only allows 1-6) - if
order=0 bounds checking is done even if checkbounds=False.
"""
# lonsin and latsin must be monotonically increasing.
if lonsin[-1] - lonsin[0] < 0 or latsin[-1] - latsin[0] < 0:
raise ValueError, 'lonsin and latsin must be increasing!'
# optionally, check that lonsout,latsout are
# within region defined by lonsin,latsin.
# (this check is always done if nearest neighbor
# interpolation (order=0) requested).
if checkbounds or order == 0:
if min(N.ravel(lonsout)) < min(lonsin) or \
max(N.ravel(lonsout)) > max(lonsin) or \
min(N.ravel(latsout)) < min(latsin) or \
max(N.ravel(latsout)) > max(latsin):
raise ValueError, 'latsout or lonsout outside range of latsin or lonsin'
# compute grid coordinates of output grid.
delon = lonsin[1:] - lonsin[0:-1]
delat = latsin[1:] - latsin[0:-1]
if max(delat) - min(delat) < 1.e-4 and max(delon) - min(delon) < 1.e-4:
# regular input grid.
xcoords = (len(lonsin) - 1) * (lonsout - lonsin[0]) / (lonsin[-1] -
lonsin[0])
ycoords = (len(latsin) - 1) * (latsout - latsin[0]) / (latsin[-1] -
latsin[0])
else:
# irregular (but still rectilinear) input grid.
lonsoutflat = N.ravel(lonsout)
latsoutflat = N.ravel(latsout)
ix = N.searchsorted(lonsin, lonsoutflat) - 1
iy = N.searchsorted(latsin, latsoutflat) - 1
xcoords = N.zeros(ix.shape, 'f')
ycoords = N.zeros(iy.shape, 'f')
for n, i in enumerate(ix):
if i < 0:
xcoords[n] = -1 # outside of range on lonsin (lower end)
elif i >= len(lonsin) - 1:
xcoords[n] = len(lonsin) # outside range on upper end.
else:
xcoords[n] = float(i) + (lonsoutflat[n] - lonsin[i]) / (
lonsin[i + 1] - lonsin[i])
xcoords = N.reshape(xcoords, lonsout.shape)
for m, j in enumerate(iy):
if j < 0:
ycoords[m] = -1 # outside of range of latsin (on lower end)
elif j >= len(latsin) - 1:
ycoords[m] = len(latsin) # outside range on upper end
else:
ycoords[m] = float(j) + (latsoutflat[m] - latsin[j]) / (
latsin[j + 1] - latsin[j])
ycoords = N.reshape(ycoords, latsout.shape)
coords = [ycoords, xcoords]
# interpolate to output grid using numarray.nd_image spline filter.
if order:
return nd_image.map_coordinates(datain,
coords,
mode=mode,
cval=cval,
order=order)
else:
# nearest neighbor interpolation if order=0.
# uses index arrays, so first convert to numarray.
datatmp = N.array(datain, datain.typecode())
xi = N.around(xcoords).astype('i')
yi = N.around(ycoords).astype('i')
return datatmp[yi, xi]
def train(self, train_toks, **kwargs):
"""
Train a new C{ConditionalExponentialClassifier}, using the
given training samples. This
C{ConditionalExponentialClassifier} should encode the model
that maximizes entropy from all the models that are
emperically consistant with C{train_toks}.
@param kwargs: Keyword arguments.
- C{iterations}: The maximum number of times IIS should
iterate. If IIS converges before this number of
iterations, it may terminate. Default=C{20}.
(type=C{int})
- C{debug}: The debugging level. Higher values will cause
more verbose output. Default=C{0}. (type=C{int})
- C{classes}: The set of possible classes. If none is given,
then the set of all classes attested in the training data
will be used instead. (type=C{list} of (immutable)).
- C{accuracy_cutoff}: The accuracy value that indicates
convergence. If the accuracy becomes closer to one
than the specified value, then IIS will terminate. The
default value is None, which indicates that no accuracy
cutoff should be used. (type=C{float})
- C{delta_accuracy_cutoff}: The change in accuracy should be
taken to indicate convergence. If the accuracy changes by
less than this value in a single iteration, then IIS will
terminate. The default value is C{None}, which indicates
that no accuracy-change cutoff should be
used. (type=C{float})
- C{log_likelihood_cutoff}: specifies what log-likelihood
value should be taken to indicate convergence. If the
log-likelihod becomes closer to zero than the specified
value, then IIS will terminate. The default value is
C{None}, which indicates that no log-likelihood cutoff
should be used. (type=C{float})
- C{delta_log_likelihood_cutoff}: specifies what change in
log-likelihood should be taken to indicate convergence.
If the log-likelihood changes by less than this value in a
single iteration, then IIS will terminate. The default
value is C{None}, which indicates that no
log-likelihood-change cutoff should be used. (type=C{float})
"""
assert _chktype(1, train_toks, [Token], (Token,))
# Process the keyword arguments.
iter = 20
debug = 0
classes = None
ll_cutoff = lldelta_cutoff = None
acc_cutoff = accdelta_cutoff = None
for (key, val) in kwargs.items():
if key in ('iterations', 'iter'): iter = val
elif key == 'debug': debug = val
elif key == 'classes': classes = val
elif key == 'log_likelihood_cutoff':
ll_cutoff = abs(val)
elif key == 'delta_log_likelihood_cutoff':
lldelta_cutoff = abs(val)
elif key == 'accuracy_cutoff':
acc_cutoff = abs(val)
elif key == 'delta_accuracy_cutoff':
accdelta_cutoff = abs(val)
else: raise TypeError('Unknown keyword arg %s' % key)
if classes is None:
classes = attested_classes(train_toks)
self._classes = classes
# Find the classes, if necessary.
if classes is None:
classes = find_classes(train_toks)
# Find the length of the first token's feature vector.
if len(train_toks) == 0:
raise ValueError('Expected at least one training token')
vector0 = train_toks[0]['FEATURE_VECTOR']
self._feature_vector_len = len(vector0)
self._weight_vector_len = self._feature_vector_len*len(self._classes)
# Build the offsets dictionary. This maps from a class to the
# index in the weight vector where that class's weights begin.
self._offsets = dict([(cls, i*self._feature_vector_len)
for i, cls in enumerate(classes)])
# Find the frequency with which each feature occurs in the
# training data.
ffreq_emperical = self._ffreq_emperical(train_toks)
# Find the nf map, and related variables nfarray and nfident.
# nf is the sum of the features for a given labeled text.
# nfmap compresses this sparse set of values to a dense list.
# nfarray performs the reverse operation. nfident is
# nfarray multiplied by an identity matrix.
nfmap = self._nfmap(train_toks)
nfs = nfmap.items()
nfs.sort(lambda x,y:cmp(x[1],y[1]))
nfarray = numarray.array([nf for (nf, i) in nfs], 'd')
nftranspose = numarray.reshape(nfarray, (len(nfarray), 1))
# An array that is 1 whenever ffreq_emperical is zero. In
# other words, it is one for any feature that's not attested
# in the data. This is used to avoid division by zero.
unattested = numarray.zeros(self._weight_vector_len, 'd')
for i in range(len(unattested)):
if ffreq_emperical[i] == 0: unattested[i] = 1
# Build the classifier. Start with weight=1 for each feature,
# except for the unattested features. Start those out at
# zero, since we know that's the correct value.
weights = numarray.ones(self._weight_vector_len, 'd')
weights -= unattested
classifier = ConditionalExponentialClassifier(classes, weights)
if debug > 0: print ' ==> Training (%d iterations)' % iter
if debug > 2:
print
print ' Iteration Log Likelihood Accuracy'
print ' ---------------------------------------'
# Train for a fixed number of iterations.
for iternum in range(iter):
if debug > 2:
print (' %9d %14.5f %9.3f' %
(iternum, classifier_log_likelihood(classifier, train_toks),
classifier_accuracy(classifier, train_toks)))
# Calculate the deltas for this iteration, using Newton's method.
deltas = self._deltas(train_toks, classifier, unattested,
ffreq_emperical, nfmap, nfarray,
nftranspose)
# Use the deltas to update our weights.
weights = classifier.weights()
weights *= numarray.exp(deltas)
classifier.set_weights(weights)
# Check log-likelihood cutoffs.
if ll_cutoff is not None or lldelta_cutoff is not None:
ll = classifier_log_likelihood(classifier, train_toks)
if ll_cutoff is not None and ll > -ll_cutoff: break
if lldelta_cutoff is not None:
if (ll - ll_old) < lldelta_cutoff: break
ll_old = ll
# Check accuracy cutoffs.
if acc_cutoff is not None or accdelta_cutoff is not None:
acc = classifier_accuracy(classifier, train_toks)
if acc_cutoff is not None and acc < acc_cutoff: break
if accdelta_cutoff is not None:
if (acc_old - acc) < accdelta_cutoff: break
acc_old = acc
if debug > 2:
print (' %9d %14.5f %9.3f' %
(iternum+1, classifier_log_likelihood(classifier, train_toks),
classifier_accuracy(classifier, train_toks)))
print
# Return the classifier.
return classifier
def __call__(self, lon, lat, inverse=False):
"""
Calling a Proj class instance with the arguments lon, lat will
convert lon/lat (in degrees) to x/y native map projection
coordinates (in meters). If optional keyword 'inverse' is
True (default is False), the inverse transformation from x/y
to lon/lat is performed.
Example usage:
>>> from proj import Proj
>>> params = {}
>>> params['proj'] = 'lcc' # lambert conformal conic
>>> params['lat_1'] = 30. # standard parallel 1
>>> params['lat_2'] = 50. # standard parallel 2
>>> params['lon_0'] = -96 # central meridian
>>> map = Proj(params)
>>> x,y = map(-107,40) # find x,y coords of lon=-107,lat=40
>>> print x,y
-92272.1004461 477477.890988
lon,lat can be either scalar floats or numarray arrays.
"""
npts = 1
# if inputs are numarray arrays, get shape and total number of pts.
try:
shapein = lon.shape
npts = reduce(lambda x, y: x * y, shapein)
lontypein = lon.typecode()
lattypein = lat.typecode()
except:
shapein = False
# make sure inputs have same shape.
if shapein and lat.shape != shapein:
raise ValueError, 'lon, lat must be scalars or numarray arrays with the same shape'
# make a rank 1 array with x/y (or lon/lat) pairs.
xy = numarray.zeros(2 * npts, 'd')
xy[::2] = numarray.ravel(lon)
xy[1::2] = numarray.ravel(lat)
# convert degrees to radians, meters to cm.
if not inverse:
xy = _dg2rad * xy
else:
xy = 10. * xy
# write input data to binary file.
xyout = array.array('d')
xyout.fromlist(xy.tolist())
# set up cmd string, scale factor for output.
if inverse:
cmd = self.projcmd + ' -I'
scale = _rad2dg
else:
cmd = self.projcmd
scale = 0.1
# use pipes for input and output if amount of
# data less than default buffer size (usually 8192).
# otherwise use pipe for input, tempfile for output
if 2 * npts * 8 > 8192:
fd, fn = tempfile.mkstemp()
os.close(fd)
stdout = open(fn, 'rb')
cmd = cmd + ' > ' + fn
stdin = os.popen(cmd, 'w')
xyout.tofile(stdin)
stdin.close()
outxy = scale * numarray.fromstring(stdout.read(), 'd')
stdout.close()
os.remove(fn)
else:
stdin, stdout = os.popen2(cmd, mode='b')
xyout.tofile(stdin)
stdin.close()
outxy = scale * numarray.fromstring(stdout.read(), 'd')
stdout.close()
# return numarray arrays or scalars, depending on type of input.
if shapein:
outx = numarray.reshape(outxy[::2], shapein).astype(lontypein)
outy = numarray.reshape(outxy[1::2], shapein).astype(lattypein)
else:
outx = outxy[0]
outy = outxy[1]
return outx, outy