def checkAll(self):
""" Method to check the dtd structure to see if all expected
nodes were found.
Parameters
----------
None
Returns
-------
Boolean, whether or not all nodes were found
"""
#pp.pprint(self.entities)
for i in self.entities:
if not self.entities[i]["found"]:
print self.xmlFile
logging.info(str(i) + " not found")
return False
for a in self.entities[i]["attrib"]:
if not self.entities[i]["attrib"][a]["found"]:
print "2",self.xmlFile
logging.info(str(i) + " " + str(a) + " not found")
return False
return True
def set_mask(self, mask):
""" Method to set the mask item
Parameters
----------
mask : array like or bool
The mask to use, must be equal in dimmension to
the spectral axis, or a single boolean value which
is applied to all data.
Returns
-------
None
"""
tempmask = self._mask
if isinstance(mask, ma.masked_array):
if len(self._spec) > len(mask) > 1:
raise
elif len(mask) == 1:
self._mask = np.array([mask.data[0]] * len(self._spec))
else:
self._mask = np.array(mask.data)
elif isinstance(mask, list) or isinstance(mask, np.ndarray):
if len(self._spec) > len(mask) > 1:
raise
elif len(mask) == 1:
self._mask = np.array([mask[0]] * len(self._spec))
else:
self._mask = np.array(mask)
if self.integritycheck():
return
logging.warning(" Mask not applied")
self._mask = tempmask
def mask_outside(self, limit1, limit2, axis="spec"):
""" Method to mask any data less than or greater than the given values.
Any axis can be used ("spec", "freq", "chans") as
the basis for the masking.
Parameters
----------
limit1 : float or int
The value below which all data of the given axis
will be masked.
limit2 : float or int
The value above which all data of the given axis
will be masked.
axis : str
The axis which is used to determine the flags.
Default: "spec"
Returns
-------
None
"""
if not self.integritycheck():
logging.warning(" Masking operation not performed.")
return
mask1 = getattr(self, "_" + axis, None) < limit1
mask2 = limit2 < getattr(self, "_" + axis, None)
mask = np.logical_and(mask1, mask2)
self._mask = np.logical_or(self._mask, mask)
def test_warning(self):
msg = "unit_test_warning_message"
Alogging.warning(msg)
found = False
r = open(self.logfile, 'r')
for line in r.readlines():
if msg in line:
if(self.verbose):
print "\nFound message >", line
found = True
r.close()
break
# since this is the last test case, we now close off the logging
Alogging.shutdown()
try:
if os.path.exists(self.logfile):
os.remove(self.logfile)
except RuntimeError:
pass
self.assertTrue(found)
def admit_root(path = None):
""" Return the root directory of the ADMIT environment
Typically in ADMIT/etc there are files needed by certain functions
and other TBD locations.
For now we are using getenv(), but in deployment this may not
be the case.
Parameters
----------
path : string
Optional path appended to the ADMIT root
Returns
-------
String containing the absolute address of the admit root directory,
with the optional path appended
"""
global _admit_root
if _admit_root != None: return _admit_root
# try the old style developer environment first
_admit_root = os.getenv("ADMIT")
if _admit_root == None:
# try the dumb generic way; is that safe to reference from __file__ ???
_admit_root = version.__file__.rsplit('/',2)[0]
if _admit_root[0] != '/':
logging.warning("ADMIT_ROOT is a relative address")
# @tdodo shouldn't that be a fatal error
#
print "_ADMIT_ROOT=",_admit_root
if path == None:
return _admit_root
return _admit_root + '/' + path
def mask_ge(self, limit, axis="spec"):
""" Method to mask any data less than or equal to the given value.
Any axis can be used ("spec", "freq", "chans") as
the basis for the masking.
Parameters
----------
limit : float or int
The value which all data, of the given axis,
greater than or equal to, will be masked.
axis : str
The axis which is used to determine the flags.
Default: "spec"
Returns
-------
None
"""
if not self.integritycheck():
logging.warning(" Masking operation not performed.")
return
mask = getattr(self, "_" + axis, None) >= limit
self._mask = np.logical_or(self._mask, mask)
def mask_ge(self, limit, axis="spec"):
""" Method to mask any data less than or equal to the given value.
Any axis can be used ("spec", "freq", "chans") as
the basis for the masking.
Parameters
----------
limit : float or int
The value which all data, of the given axis,
greater than or equal to, will be masked.
axis : str
The axis which is used to determine the flags.
Default: "spec"
Returns
-------
None
"""
if not self.integritycheck():
logging.warning(" Masking operation not performed.")
return
mask = getattr(self, "_" + axis, None) >= limit
self._mask = np.logical_or(self._mask, mask)
def addBDPtoAT(self, bdp):
""" Method to add a BDP to an AT. The AT is not specified, but the
_taskid attribute of the BDP is used to identify the necessary AT.
Parameters
----------
bdp : BDP
Any valid BDP, to be added to an existing AT.
Returns
-------
None
"""
found = False
cp = copy.deepcopy(bdp)
# find the AT we need
for at in self.tasks:
# see if the ID's match
if at._taskid == bdp._taskid:
found = True
# set the base directory of the BDP
cp.baseDir(at.baseDir())
# add it to the correct slot
at._bdp_out[at._bdp_out_map.index(cp._uid)] = cp
break
if not found:
logging.info("##### Found orphaned BDP with type %s in file %s" % \
(bdp._type, bdp.xmlFile))
def admit_root(path=None):
""" Return the root directory of the ADMIT environment
Typically in ADMIT/etc there are files needed by certain functions
and other TBD locations.
For now we are using getenv(), but in deployment this may not
be the case.
Parameters
----------
path : string
Optional path appended to the ADMIT root
Returns
-------
String containing the absolute address of the admit root directory,
with the optional path appended
"""
global _admit_root
if _admit_root != None: return _admit_root
# try the old style developer environment first
_admit_root = os.getenv("ADMIT")
if _admit_root == None:
# try the dumb generic way; is that safe to reference from __file__ ???
_admit_root = version.__file__.rsplit('/', 2)[0]
if _admit_root[0] != '/':
logging.warning("ADMIT_ROOT is a relative address")
# @tdodo shouldn't that be a fatal error
#
print "_ADMIT_ROOT=", _admit_root
if path == None:
return _admit_root
return _admit_root + '/' + path
def end(self):
t0 = self.init
t1 = self.time()
dt = t1 - t0
if self.report:
logging.timing("%s END " % self.label + str(dt))
return dt
def set_mask(self, mask):
""" Method to set the mask item
Parameters
----------
mask : array like or bool
The mask to use, must be equal in dimmension to
the spectral axis, or a single boolean value which
is applied to all data.
Returns
-------
None
"""
tempmask = self._mask
if isinstance(mask, ma.masked_array):
if len(self._spec) > len(mask) > 1:
raise
elif len(mask) == 1:
self._mask = np.array([mask.data[0]] * len(self._spec))
else:
self._mask = np.array(mask.data)
elif isinstance(mask, list) or isinstance(mask, np.ndarray):
if len(self._spec) > len(mask) > 1:
raise
elif len(mask) == 1:
self._mask = np.array([mask[0]] * len(self._spec))
else:
self._mask = np.array(mask)
if self.integritycheck():
return
logging.warning(" Mask not applied")
self._mask = tempmask
def end(self):
t0 = self.init
t1 = self.time()
dt = t1 - t0
if self.report:
logging.timing("%s END " % self.label + str(dt))
return dt
def mask_outside(self, limit1, limit2, axis="spec"):
""" Method to mask any data less than or greater than the given values.
Any axis can be used ("spec", "freq", "chans") as
the basis for the masking.
Parameters
----------
limit1 : float or int
The value below which all data of the given axis
will be masked.
limit2 : float or int
The value above which all data of the given axis
will be masked.
axis : str
The axis which is used to determine the flags.
Default: "spec"
Returns
-------
None
"""
if not self.integritycheck():
logging.warning(" Masking operation not performed.")
return
mask1 = getattr(self, "_" + axis, None) < limit1
mask2 = limit2 < getattr(self, "_" + axis, None)
mask = np.logical_and(mask1, mask2)
self._mask = np.logical_or(self._mask, mask)
def get_mem(self):
""" Read memory usage info from /proc/pid/status
Return Virtual and Resident memory size in MBytes.
"""
global ostype
if ostype == None:
ostype = os.uname()[0].lower()
logging.info("OSTYPE: %s" % ostype)
scale = {'MB': 1024.0}
lines = []
try:
if ostype == 'linux':
proc_status = '/proc/%d/status' % os.getpid() # linux only
# open pseudo file /proc/<pid>/status
t = open(proc_status)
# get value from line e.g. 'VmRSS: 9999 kB\n'
for it in t.readlines():
if 'VmSize' in it or 'VmRSS' in it :
lines.append(it)
t.close()
else:
proc = subprocess.Popen(['ps','-o', 'rss', '-o', 'vsz', '-o','pid', '-p',str(os.getpid())],stdout=subprocess.PIPE)
proc_output = proc.communicate()[0].split('\n')
proc_output_memory = proc_output[1]
proc_output_memory = proc_output_memory.split()
phys_mem = int(proc_output_memory[0])/1204 # to MB
virtual_mem = int(proc_output_memory[1])/1024
except (IOError, OSError):
if self.report:
logging.timing(self.label + " Error: cannot read memory usage information.")
return np.array([])
# parse the two lines
mem = {}
if(ostype != 'darwin'):
for line in lines:
words = line.strip().split()
#print words[0], '===', words[1], '===', words[2]
# get rid of the tailing ':'
key = words[0][:-1]
# convert from KB to MB
scaled = float(words[1]) / scale['MB']
mem[key] = scaled
else:
mem['VmSize'] = virtual_mem
mem['VmRSS'] = phys_mem
return np.array([mem['VmSize'], mem['VmRSS']])
def __init__(self, upper=True):
self.version = "27-apr-2016"
if have_ADMIT:
self.table = utils.admit_root() + "/etc/vlsr.tab"
self.cat = read_vlsr(self.table,upper)
logging.debug("VLSR: %s, found %d entries" % (self.table,len(self.cat)))
else:
logging.warning("VLSR: Warning, no ADMIT, empty catalogue")
self.cat = {}
def __init__(self, label=".", report=True):
self.start = self.time()
self.init = self.start
self.label = label
self.report = report
self.dtimes = []
dt = self.init - self.init
if self.report:
logging.timing("%s ADMIT " % self.label + str(self.start))
logging.timing("%s BEGIN " % self.label + str(dt))
def __init__(self, label=".", report=True):
self.start = self.time()
self.init = self.start
self.label = label
self.report = report
self.dtimes = []
dt = self.init - self.init
if self.report:
logging.timing("%s ADMIT " % self.label + str(self.start))
logging.timing("%s BEGIN " % self.label + str(dt))
def tab_to_slit(xym, clip=0.0, gamma=1.0):
"""take all values from a map over clip, compute best slit for PV Slice
"""
x = xym[0] # maxposx
y = xym[1] # maxposy
m = xym[2] # max
logging.debug("CLIP %g" % clip)
slit = convert_to_slit(m,x,y,0,0,gamma,expand=2.0)
return (slit,clip)
def error(self, ex):
""" Method called when an error is encountered
Parameters
----------
ex : exception
The error that was encountered
Returns
-------
None
"""
logging.warning("Recoverable error encountered: %s" % (ex.message))
def warning(self, ex):
""" Method called when the parser issues a warning
Parameters
----------
ex : exception
The warning to be issued.
Returns
-------
None
"""
logging.warning("Warning: %s" % (ex.message))
def peakstats(image, freq, sigma, nsigma, minchan, maxgap, psample, peakfit = False):
""" Go through a cube and find peaks in the spectral dimension
It will gather a table of <peak>,<freq>,<sigma> which can be
optionally used for plotting
"""
if psample < 0: return
cutoff = nsigma * sigma
madata = casautil.getdata(image)
data = madata.data
shape = data.shape
logging.debug("peakstats: shape=%s cutoff=%g" % (str(shape),cutoff))
#print "DATA SHAPE:",shape
#print "cutoff=",cutoff
nx = shape[0]
ny = shape[1]
nz = shape[2]
chan = np.arange(nz)
# prepare the segment finder
# we now have an array data[nx,ny,nz]
sum = 0.0
pval = []
mval = []
wval = []
for x in range(0,nx,psample):
for y in range(0,ny,psample):
s0 = data[x,y,:]
spec = ma.masked_invalid(s0)
sum += spec.sum()
# using abs=True is a bit counter intuitive, but a patch to deal with the confusion in
# ADMITSegmentFinder w.r.t abs usage
asf = ADMITSegmentFinder(pmin=nsigma, minchan=minchan, maxgap=maxgap, freq=freq, spec=spec, abs=True)
#asf = ADMITSegmentFinder(pmin=nsigma, minchan=minchan, maxgap=maxgap, freq=freq, spec=spec, abs=False)
f = asf.line_segments(spec, nsigma*sigma)
for s in f:
if False:
for i in range(s[0],s[1]+1):
print "# ",x,y,i,spec[i]
## area preserving and peak are correlated, 18% difference
## fitgauss1Dm was about 5"
## with fitgauss1D was about 30", and still bad fits
par = utils.fitgauss1Dm(chan[s[0]:s[1]+1], spec[s[0]:s[1]+1], True) # peak from max
#par = utils.fitgauss1Dm(chan[s[0]:s[1]+1], spec[s[0]:s[1]+1], False) # peak from area preserving
if peakfit:
(par,cov) = utils.fitgauss1D (chan[s[0]:s[1]+1], spec[s[0]:s[1]+1],par)
#print "FIND: ",x,y,s,cutoff,0.0,0.0,par[0],par[1],par[2],s[1]-s[0]+1
pval.append(par[0])
mval.append(par[1])
wval.append(par[2])
#print "SUM:",sum
return (np.array(pval),np.array(mval),np.array(wval))
def fatalError(self, ex):
""" Method called when a fata error is encountered
Parameters
----------
ex : exception
The error encountered
Returns
-------
None
"""
logging.error("Fatal error encountered.")
raise ex
def fitgauss1D(x, y, par=None, width=-1.0):
""" Method for fitting a 1D gaussian to a spectral line
Parameters
----------
x: array like
The x co-ordinates of the spectrum, note the center of
the spectral line should be near 0.0 if possible
y: array like
The y co-ordinates (intensity) of the spectrum
par: array like
The initial guesses for the fit parameters, the fitter works best
if the center parameter is near 0.0
3 parameters: PeakY, CenterX, FWHM.
width: float
If positive, this is the assumed width (or step) in the x array,
which is needed if only 1 point is given. Otherwise ignored.
Returns
-------
A tuple containing the best fit parameters (as a list) and the covariance
of the parameters (also as a list)
"""
if len(x) == 3:
logging.info(
"Gaussian fit attempted with only three points, look at the covariance for goodness of fit."
)
# if there are too few points to fit then just conserve the are of the channels to calculate the
# parameters
if len(x) < 3:
logging.info(
"Gaussian fit attempted with fewer than three points (%d). Using conservation of area method to determine parameters."
% len(x))
params = fitgauss1Dm(x, y, dx=width)
covar = [1000.] * len(params)
else:
try:
params, covar = curve_fit(gaussian1D, x, y, p0=par)
# if the covariance cannot be determined, just return the initial values
except RuntimeError, e:
if "Optimal" in str(e):
params = par
covar = [0] * len(par)
# otherwise re-raise the exception
else:
raise
def test_debug(self):
msg = "unit_test_debug_message"
Alogging.debug(msg)
found = False
r = open(self.logfile, 'r')
for line in r.readlines():
if msg in line:
if(self.verbose):
print "\nFound message > ", line
found = True
r.close()
break
self.assertTrue(found)
def tag(self, mytag):
t0 = self.start
t1 = self.time()
dt = t1 - t0
# get memory usage (Virtual and Resident) info
mem = self.get_mem()
if mem.size != 0:
dt = np.append(dt, mem)
self.dtimes.append((mytag, dt))
self.start = t1
if self.report:
logging.timing("%s " % self.label + mytag + " " + str(dt))
return dt
def tag(self, mytag):
t0 = self.start
t1 = self.time()
dt = t1 - t0
# get memory usage (Virtual and Resident) info
mem = self.get_mem()
if mem.size != 0 :
dt = np.append(dt, mem)
self.dtimes.append((mytag, dt))
self.start = t1
if self.report:
logging.timing("%s " % self.label + mytag + " " + str(dt))
return dt
def test_reportKeywords(self):
kw = {"input": "helloWorld",
"list" : [1,2,3,4]}
Alogging.reportKeywords(kw)
found = []
r = open(self.logfile, 'r')
for line in r.readlines():
for k, v in kw.iteritems():
if k in line and str(v) in line:
if(self.verbose):
print "\nFound message >", line
found.append(True)
r.close()
self.assertTrue(len(found) == 2)
def fitgauss1D(x, y, par=None, width=-1.0):
""" Method for fitting a 1D gaussian to a spectral line
Parameters
----------
x: array like
The x co-ordinates of the spectrum, note the center of
the spectral line should be near 0.0 if possible
y: array like
The y co-ordinates (intensity) of the spectrum
par: array like
The initial guesses for the fit parameters, the fitter works best
if the center parameter is near 0.0
3 parameters: PeakY, CenterX, FWHM.
width: float
If positive, this is the assumed width (or step) in the x array,
which is needed if only 1 point is given. Otherwise ignored.
Returns
-------
A tuple containing the best fit parameters (as a list) and the covariance
of the parameters (also as a list)
"""
if len(x) == 3:
logging.info("Gaussian fit attempted with only three points, look at the covariance for goodness of fit.")
# if there are too few points to fit then just conserve the are of the channels to calculate the
# parameters
if len(x) < 3:
logging.info("Gaussian fit attempted with fewer than three points (%d). Using conservation of area method to determine parameters." % len(x))
params = fitgauss1Dm(x,y,dx=width)
covar = [1000.] * len(params)
else:
try:
params, covar = curve_fit(gaussian1D, x, y, p0=par)
# if the covariance cannot be determined, just return the initial values
except RuntimeError, e:
if "Optimal" in str(e):
params = par
covar = [0] * len(par)
# otherwise re-raise the exception
else:
raise
def test_effectiveLevel(self):
msg = "unit_test_levels_message"
# check that the logging level is what is expected
level = Alogging.getEffectiveLevel()
self.assertTrue(level == self.level)
# set the level to a new value and check again
Alogging.setLevel(50)
level = Alogging.getEffectiveLevel()
self.assertTrue(level == 50)
# log an info message which is below the logging level, this message should not appear
# in the logs
Alogging.info(msg)
found = False
r = open(self.logfile, 'r')
for line in r.readlines():
if msg in line:
if(self.verbose):
print "\nFound message >", line
found = True
break
r.close()
self.assertFalse(found)
Alogging.setLevel(self.level)
# reset the logging level
msg += "2"
# log an info message, which is now above the logging level, this message should appear
# in the logs
Alogging.info(msg)
found = False
r = open(self.logfile, 'r')
for line in r.readlines():
if msg in line:
if(self.verbose):
print "\nFound message >", line
found = True
r.close()
break
self.assertTrue(found)
def mask_invalid(self):
""" Method to mask all invalid spectral data.
Invalid values are: NaN, Inf, -Inf
Parameters
----------
None
Returns
-------
None
"""
if not self.integritycheck():
logging.warning(" Masking operation not performed.")
return
mask = np.isfinite(self._spec)
self._mask = np.logical_or(self._mask, np.logical_not(mask))
def mask_invalid(self):
""" Method to mask all invalid spectral data.
Invalid values are: NaN, Inf, -Inf
Parameters
----------
None
Returns
-------
None
"""
if not self.integritycheck():
logging.warning(" Masking operation not performed.")
return
mask = np.isfinite(self._spec)
self._mask = np.logical_or(self._mask, np.logical_not(mask))
def fitgauss1Dm(xdat, ydat, usePeak=False, dx=-1.0):
""" gaussfit helper function
this will get a reasonable gauss even if you only have 2 points
assumes evenly spaced data in xdat, so it can extract a width.
It can be used like fitgauss1D(), but uses the first three moments
of the distribution to "match" that of a gauss. area preserving
if you wish.
If you set usePeak, it will pick the highest value in your data.
Warning: if you must fit negative profiles, be sure to rescale
before you come into this routine.
"""
if len(xdat) == 1:
# special case, it will need dx
if dx < 0.0:
logging.critical(
"Cannot determine gaussian of delta function if width not given"
)
raise
return (ydat[0], xdat[0], dx)
sum0 = sum1 = sum2 = peak = 0.0
# the mean is given as the weighted mean of x
for x, y in zip(xdat, ydat):
if y > peak: peak = y
sum0 = sum0 + y
sum1 = sum1 + y * x
xmean = sum1 / sum0
# re-center the data for a moment-2 calculation
# @todo pos/neg
xdat0 = xdat - xmean
for x, y in zip(xdat0, ydat):
sum2 = sum2 + y * x * x
sigma = math.sqrt(abs(sum2 / sum0))
# equate the area under the histogram with the area under a gauss
# to get the peak of the "matched" gauss
dx = abs(xdat[1] - xdat[0]) # @todo use optional "dx=None" ?
if not usePeak:
# pick the area preserving one, vs. the "real" peak
# The area preserving one seems to be about 18% higher
# but with a tight correllation
peak = (sum0 * dx) / math.sqrt(2 * sigma * math.pi)
fwhm = 2.35482 * sigma
return (peak, xmean, fwhm)
def fitgauss1Dm(xdat, ydat, usePeak = False, dx = -1.0):
""" gaussfit helper function
this will get a reasonable gauss even if you only have 2 points
assumes evenly spaced data in xdat, so it can extract a width.
It can be used like fitgauss1D(), but uses the first three moments
of the distribution to "match" that of a gauss. area preserving
if you wish.
If you set usePeak, it will pick the highest value in your data.
Warning: if you must fit negative profiles, be sure to rescale
before you come into this routine.
"""
if len(xdat) == 1:
# special case, it will need dx
if dx < 0.0:
logging.critical("Cannot determine gaussian of delta function if width not given")
raise
return (ydat[0], xdat[0], dx)
sum0 = sum1 = sum2 = peak = 0.0
# the mean is given as the weighted mean of x
for x,y in zip(xdat,ydat):
if y>peak: peak = y
sum0 = sum0 + y
sum1 = sum1 + y*x
xmean = sum1/sum0
# re-center the data for a moment-2 calculation
# @todo pos/neg
xdat0 = xdat - xmean
for x,y in zip(xdat0,ydat):
sum2 = sum2 + y*x*x
sigma = math.sqrt(abs(sum2/sum0))
# equate the area under the histogram with the area under a gauss
# to get the peak of the "matched" gauss
dx = abs(xdat[1]-xdat[0]) # @todo use optional "dx=None" ?
if not usePeak:
# pick the area preserving one, vs. the "real" peak
# The area preserving one seems to be about 18% higher
# but with a tight correllation
peak = (sum0 * dx) / math.sqrt(2*sigma*math.pi)
fwhm = 2.35482 * sigma
return (peak, xmean, fwhm)
def setUp(self):
self.verbose = False
self.testName = "Utility AdmitLogging Class Unit Test"
self.logfile = '/tmp/logging_unit_test_%s.log' % os.getpid()
self.level = 1
# only need to initialize the logger once since it is a static class
if not TestAdmitLogging.setup:
Alogging.init(name="test", logfile=self.logfile, level=Alogging.DEBUG)
Alogging.addLevelName(15, "TIMING")
Alogging.addLevelName(16, "REGRSSION")
Alogging.setLevel(self.level)
TestAdmitLogging.setup = True
def checkAttribute(self, name, attrib, value=None):
""" Method to check an attribute for validity. Validity includes correct
name and data type.
Parameters
----------
name : str
The name of the node being checked
attrib : str
The attribute of the node being checked, if any.
value : str
The type of the attribute being checked (e.g. bt.INT)
Default: None
"""
# check an attribute for validity
try:
if not value in self.entities[name]["attrib"][attrib]["values"] \
and not "ANY" in self.entities[name]["attrib"][attrib]["values"]:
raise Exception("DTDParser.checkAttributes: Value %s for attribute %s is not a valid entry (file %s)" %
(value, name, self.xmlFile))
self.entities[name]["attrib"][attrib]["found"] = True
except KeyError:
logging.info("Attribute %s not listed in DTD, malformed xml detected (%s)" %
(attrib, self.xmlFile))
logging.info("Inconsistency between dtd and xml detected, continuing")
except:
logging.info("Unknown error encountered while parsing attribute %s (%s)" %
(attrib, self.xmlFile))
raise
def integritycheck(self):
""" Method to check that all axes are the same length. Axes
that have no data are ignored.
Parameters
----------
None
Returns
-------
Bool, True if all match, False if there is an inconsistency
"""
if self._spec is not None:
ls = len(self._spec)
else:
ls = 0
if self._freq is not None:
lf = len(self._freq)
else:
lf = ls
if self._chans is not None:
lc = len(self._chans)
else:
lc = ls
if self._mask is not None:
lm = len(self._mask)
else:
lm = ls
if self._contin is not None:
lco = len(self._contin)
else:
lco = ls
if ls == lf == lc == lm == lco:
return True
logging.warning(
"Inconsistent axes: spectrum: %i, freq: %i, chans: %i, mask: %i, contin: %i"
% (ls, lf, lc, lm, lco))
return False
def fix_invalid(self, mask_value=0.0):
""" Method to replace invalid spectral data with a specific value
and mask it. Invalid values are: NaN, Inf, -Inf
Parameters
----------
mask_value : float
The value to replace the invalid data with.
Default: 0.0
Returns
-------
None
"""
if not self.integritycheck():
logging.warning(" Masking operation not performed.")
return
mask = np.isfinite(self._spec)
for i in range(len(mask)):
if not mask[i]:
self._spec[i] = mask_value
self._mask = np.logical_or(self._mask, np.logical_not(mask))
def run(self):
""" running the File_AT task
"""
# grab and check essential keywords
filename = self.getkey('file')
logging.info("file=%s" % filename)
if len(filename) == 0:
raise Exception,'File_AT: no file= given'
exist = self.getkey('exist')
if exist:
#
logging.warning("no checking now")
# self._bdp_in[0].checkfiles()
# create the BDP
bdp1 = File_BDP(filename)
bdp1.filename = filename
self.addoutput(bdp1)
# touch the file if desired
if self.getkey('touch'): bdp1.touch()
def fix_invalid(self, mask_value=0.0):
""" Method to replace invalid spectral data with a specific value
and mask it. Invalid values are: NaN, Inf, -Inf
Parameters
----------
mask_value : float
The value to replace the invalid data with.
Default: 0.0
Returns
-------
None
"""
if not self.integritycheck():
logging.warning(" Masking operation not performed.")
return
mask = np.isfinite(self._spec)
for i in range(len(mask)):
if not mask[i]:
self._spec[i] = mask_value
self._mask = np.logical_or(self._mask, np.logical_not(mask))
def run(self):
""" running the File_AT task
"""
# grab and check essential keywords
filename = self.getkey('file')
logging.info("file=%s" % filename)
if len(filename) == 0:
raise Exception, 'File_AT: no file= given'
exist = self.getkey('exist')
if exist:
#
logging.warning("no checking now")
# self._bdp_in[0].checkfiles()
# create the BDP
bdp1 = File_BDP(filename)
bdp1.filename = filename
self.addoutput(bdp1)
# touch the file if desired
if self.getkey('touch'): bdp1.touch()
def integritycheck(self):
""" Method to check that all axes are the same length. Axes
that have no data are ignored.
Parameters
----------
None
Returns
-------
Bool, True if all match, False if there is an inconsistency
"""
if self._spec is not None:
ls = len(self._spec)
else:
ls = 0
if self._freq is not None:
lf = len(self._freq)
else:
lf = ls
if self._chans is not None:
lc = len(self._chans)
else:
lc = ls
if self._mask is not None:
lm = len(self._mask)
else:
lm = ls
if self._contin is not None:
lco = len(self._contin)
else:
lco = ls
if ls == lf == lc == lm == lco:
return True
logging.warning("Inconsistent axes: spectrum: %i, freq: %i, chans: %i, mask: %i, contin: %i" % (ls, lf, lc, lm, lco))
return False
def parse_response(self):
""" Parse a response (of type urllib2.urlopen) into a list of dictionaries.
Parameters
----------
None
Returns
-------
A dictonary of the response
"""
result = []
response = self.response
try:
csvstring = response.read()
tablelist = csvstring.split('\n')
header = tablelist[0]
headerlist = header.split(':')
headerlist = clean_column_headings(headerlist, renaming_dict=column_headings_map)
#Populate list of dictionaries
for row in tablelist[1:-1]:
rowlist = row.split(':')
rowdict = {}
for i in range(0, len(rowlist)):
rowdict[headerlist[i]] = rowlist[i]
result.append(rowdict)
except:
logging.warning("Problem parsing result")
self.result = result
return result
def characters(self, ch):
""" Method called whenever characters are detected in an xml node
This method does some dtd validation. This
method is only called by the SAX parser iteself.
Parameters
----------
ch : unicode characters
Returns
-------
None
"""
target = None
char = str(ch).strip()
if char.isspace() or not char:
return
# determine which class the data are getting writtrn to
if self.inUtil:
target = self.Util
elif self.inBDP:
target = self.BDP
elif self.inAT:
target = self.curAT
elif self.inSummaryEntry:
target = self.summaryEntry
elif self.inSummary:
target = self.summaryData
else:
target = self.admit
# a list or dictionary has to be decoded
if isinstance(self.type, list) or isinstance(self.type, dict) \
or isinstance(self.type, tuple) or isinstance(self.type, set) \
or isinstance(self.type, np.ndarray) or isinstance(self.type, str):
if self.inflow:
self.flowdata += char
else:
self.tempdata += char
else:
# check the version
if self.name == "_version":
ver = self.getattr(target, self.name)
vercheck = utils.compareversions(ver, str(char))
if vercheck < 0: # newer read in
logging.warning("Version mismatch for %s, data are a newer version than current software, attempting to continue." % target.getkey("_type"))
elif vercheck > 0: # older read in
logging.warning("Version mismatch for %s, data are an older version than current software, attempting to continue." % target.getkey("_type"))
else:
try:
self.setattr(target, self.name, self.getData(char))
except AttributeError:
logging.info("Data member %s is not a member of %s. This may be due to a version mismatch between the data and your software, attempting to continue." % (self.name, str(type(target))))
except:
raise
del ch
def check(self, name, attrib=None, value=None):
""" Method to check a node for validity. Validity includes correct name
and data type.
Parameters
----------
name : str
The name of the node being checked
attrib : str
The attribute of the node being checked, if any.
Default: None
value : str
The type of the attribute being checked (e.g. bt.INT)
Default: None
"""
# check a node for validity
try:
# note that the node has been found
self.entities[name]["found"] = True
# if there is an attribute specified then check it too
# if the attribute was not expected just print a note to the screen
if attrib is not None:
try:
if not value in self.entities[name]["attrib"][attrib]["values"] \
and not "ANY" in self.entities[name]["attrib"][attrib]["values"]:
raise Exception("DTDParser.check: Value %s for attribute %s is not a valid entry (attribute = %s) (file %s)" % (value, name, attrib, self.xmlFile))
self.entities[name]["attrib"][attrib]["found"] = True
except KeyError:
logging.info("Attribute %s for %s not listed in DTD, malformed xml detected (%s)" %
(attrib, name, self.xmlFile))
logging.info("Inconsistency between dtd and xml detected, continuing")
except:
logging.info("Unknown error encountered while parsing attribute %s for %s (%s)" %
(attrib, name, self.xmlFile))
raise
except KeyError:
logging.info("Data member %s is not a member of the dtd, xml inconsistent with definition (%s)" %
(name, self.xmlFile))
except:
raise
def run(self):
""" The run method creates the BDP
Parameters
----------
None
Returns
-------
None
"""
dt = utils.Dtime("Export") # tagging time
basename = self.getkey("basename")
nbdp = len(self._bdp_in)
logging.info("Found %d input BDPs" % nbdp)
if nbdp > 1:
logging.info("Only dealing with 1 BDP now")
b1 = self._bdp_in[0] # image/cube
infile = b1.getimagefile(bt.CASA) # ADMIT filename of the image (cube)
if len(basename) == 0:
fitsname = self.mkext(
infile, 'fits'
) # morph to the new output name with replaced extension '
image_out = self.dir(fitsname) # absolute filename
else:
if basename[0:2] == './' or basename[0] == '/':
image_out = basename + ".fits"
else:
image_out = self.dir(basename + ".fits")
dt.tag("start")
logging.info("Writing FITS %s" % image_out)
# @todo check self.dir(image_out)
casa.exportfits(self.dir(infile), image_out, overwrite=True)
dt.tag("done")
dt.end()
def run(self):
""" The run method creates the BDP
Parameters
----------
None
Returns
-------
None
"""
dt = utils.Dtime("Export") # tagging time
basename = self.getkey("basename")
nbdp = len(self._bdp_in)
logging.info("Found %d input BDPs" % nbdp)
if nbdp > 1:
logging.info("Only dealing with 1 BDP now")
b1 = self._bdp_in[0] # image/cube
infile = b1.getimagefile(bt.CASA) # ADMIT filename of the image (cube)
if len(basename) == 0:
fitsname = self.mkext(infile,'fits') # morph to the new output name with replaced extension '
image_out = self.dir(fitsname) # absolute filename
else:
if basename[0:2] == './' or basename[0] == '/':
image_out = basename + ".fits"
else:
image_out = self.dir(basename + ".fits")
dt.tag("start")
logging.info("Writing FITS %s" % image_out)
# @todo check self.dir(image_out)
casa.exportfits(self.dir(infile), image_out, overwrite=True)
dt.tag("done")
dt.end()
def run(self):
""" The run method, creates the slices, regrids if requested, and
creates the BDP(s)
Parameters
----------
None
Returns
-------
None
"""
dt = utils.Dtime("LineCube")
self._summary = {}
# look for an input noise level, either through keyword or input
# CubeStats BDP or calculate it if needed
pad = self.getkey("pad")
equalize = self.getkey("equalize")
minchan = 0
linelist = self._bdp_in[1]
if linelist == None or len(linelist) == 0:
logging.info("No lines found in input LineList_BDP, exiting.")
return
spw = self._bdp_in[0]
# get the columns from the table
cols = linelist.table.getHeader()
# get the casa image
imagename = spw.getimagefile(bt.CASA)
imh = imhead(self.dir(imagename), mode='list')
# set the overall parameters for imsubimage
args = {"imagename" : self.dir(imagename),
"overwrite" : True}
dt.tag("start")
if pad != 0:
nchan = imh['shape'][2]
dt.tag("pad")
# if equal size cubes are requested, this will honor the requested pad
if equalize:
start = linelist.table.getColumnByName("startchan")
end = linelist.table.getColumnByName("endchan")
# look for the widest line
for i in range(len(start)):
diff = end[i] - start[i] + 1
minchan = max(minchan , diff + (pad * 2))
dt.tag("equalize")
# get all of the rows in the table
rows = linelist.getall()
delrow = set()
procblend = [0]
# search through looking for blended lines, leave only the strongest from each blend
# in the list
for i, row in enumerate(rows):
if row.blend in procblend:
continue
strongest = -100.
index = -1
indexes = []
blend = row.blend
for j in range(i, len(rows)):
if rows[j].blend != blend:
continue
indexes.append(j)
if rows[j].linestrength > strongest:
strongest = rows[j].linestrength
index = j
indexes.remove(index)
delrow = delrow | set(indexes)
procblend.append(blend)
dr = list(delrow)
dr.sort()
dr.reverse()
for row in dr:
del rows[row]
# check on duplicate UID's, since those are the directory names here
uid1 = []
for row in rows:
uid1.append(row.getkey("uid"))
uid2 = set(uid1)
if len(uid1) != len(uid2):
print "LineList:",uid1
logging.warning("There are duplicate names in the LineList")
#raise Exception,"There are duplicate names in the LineList"
# Create Summary table
lc_description = admit.util.Table()
lc_description.columns = ["Line Name","Start Channel","End Channel","Output Cube"]
lc_description.units = ["","int","int",""]
lc_description.description = "Parameters of Line Cubes"
# loop over all entries in the line list
rdata = []
for row in rows:
uid = row.getkey("uid")
cdir = self.mkext(imagename,uid)
self.mkdir(cdir)
basefl = uid
lcd = [basefl]
outfl = cdir + os.sep + "lc.im"
args["outfile"] = self.dir(outfl)
start = row.getkey("startchan")
end = row.getkey("endchan")
diff = end - start + 1
startch = 0
if diff < minchan:
add = int(math.ceil(float(minchan - diff) / 2.0))
start -= add
end += add
startch += add
if start < 0:
logging.info("%s is too close to the edge to encompass with the "
+ "requested channels, start=%d resetting to 0" %
(uid, start))
startch += abs(start)
start = 0
if end >= nchan:
logging.info("%s is too close to the edge to encompass with the "
+ "requested channels, end=%d resetting to %d" %
(uid, end, nchan - 1))
end = nchan - 1
#print "\n\nDIFF ",startch,"\n\n"
if pad > 0 and not equalize:
start -= pad
end += pad
if start < 0:
logging.warning("pad=%d too large, start=%d resetting to 0"
% (pad, start))
startch += abs(start)
start = 0
else:
startch += pad
if end >= nchan:
logging.warning("pad=%d too large, end=%d resetting to %d"
% (pad, end, nchan - 1))
end = nchan - 1
elif pad < 0 and not equalize:
mid = (start + end) / 2
start = mid + pad / 2
end = mid - pad / 2 - 1
if start < 0:
logging.warning("pad=%d too large, start=%d resetting to 0"
% (pad, start))
startch += abs(start)
start = 0
else:
startch += abs(start)
if end >= nchan:
logging.warning("pad=%d too large, end=%d resetting to %d"
% (pad, end, nchan - 1))
end = nchan - 1
endch = startch + diff
args["chans"] = "%i~%i" % (start, end)
rdata.append(start)
rdata.append(end)
# for the summmary, which will be a table of
# Line name, start channel, end channel, output image
lc_description.addRow([basefl, start, end, outfl])
# create the slices
imsubimage(**args)
line = row.converttoline()
# set the restfrequency ouf the output cube
imhead(imagename=args["outfile"], mode="put", hdkey="restfreq",
hdvalue="%fGHz" % (row.getkey("frequency")))
# set up the output BDP
images = {bt.CASA : outfl}
casaimage = Image(images=images)
# note that Summary.getLineFluxes() implicitly relies on the BDP out order
# being the same order as in the line list table. If this is ever not
# true, then Summary.getLineFluxes mismatch BDPs and flux values.
#self.addoutput(LineCube_BDP(xmlFile=cdir + os.sep + basefl + ".lc",
self.addoutput(LineCube_BDP(xmlFile=outfl,
image=casaimage, line=line, linechans="%i~%i" % (startch, endch)))
dt.tag("trans-%s" % cdir)
logging.regression("LC: %s" % str(rdata))
taskargs = "pad=%s equalize=%s" % (pad, equalize)
self._summary["linecube"] = SummaryEntry(lc_description.serialize(), "LineCube_AT",
self.id(True), taskargs)
dt.tag("done")
dt.end()
def endElement(self, name):
""" Method called whenever the end of an xml element is reached. This
method is only called by the SAX parser iteself.
Parameters
----------
name : str
The name of the node that just ended
Returns
-------
None
"""
# reset the tracking stuff, add BDP's to AT's, AT's to the flowmanager
# reconstruct any nodes that spanned multiple lines
if name == self.utilName:
# add the utility classes to the appropriate parent class
# Images always get added to MultiImages
if self.inMulti:
self.MultiImage.addimage(copy.deepcopy(self.Util), self.Util.name)
elif self.inBDP:
setattr(self.BDP, self.utilName, copy.deepcopy(self.Util))
elif self.inAT:
setattr(self.curAT, self.utilName, copy.deepcopy(self.Util))
self.inUtil = False
self.utilName = ""
elif name == self.multiName:
if self.inBDP:
setattr(self.BDP, self.multiName, copy.deepcopy(self.MultiImage))
elif self.inAT:
setattr(self.curAT, self.multiName, copy.deepcopy(self.MultiImage))
self.multiImageName = ""
self.inMulti = False
self.inUtil = False
elif name == bt.BDP:
# one last validation run
self.BDP._baseDir = self.basedir
if not self.dtd.checkAll():
logging.info("Some required nodes missing from xml file, attempting to continue anyway.")
elif name == bt.FLOWMANAGER:
temp = aast.literal_eval(self.flowdata)
for key in ["depsmap", "varimap"]:
if key in temp:
temp[key] = eval(temp[key])
self.flowmanager = fm.FlowManager(**temp)
self.inflow = False
elif isinstance(self.type, str):
if self.inUtil:
target = self.Util
elif self.inBDP:
target = self.BDP
elif self.inAT:
target = self.curAT
elif self.inSummaryEntry:
target = self.summaryEntry
elif name == "projmanager":
target = self
else:
target = self.admit
self.setattr(target, name, self.tempdata)
self.tempdata = ""
elif isinstance(self.type, list) or isinstance(self.type, dict) \
or isinstance(self.type, tuple) or isinstance(self.type, set):
temp = aast.literal_eval(self.tempdata)
if self.inUtil:
target = self.Util
elif self.inBDP:
target = self.BDP
elif self.inAT:
target = self.curAT
elif self.inSummaryEntry:
target = self.summaryEntry
elif name == "projmanager":
target = self
else:
target = self.admit
for i in self.ndarr:
temp[i] = np.array(temp[i], dtype=object)
for i in self.sets:
temp[i] = set(temp[i])
if isinstance(self.type, tuple):
temp = tuple(temp)
elif isinstance(self.type, set):
temp = set(temp)
try:
self.setattr(target, name, temp)
except AttributeError:
logging.info("Data member %s is not a member of %s. This may be due to a version mismatch between the data and your software, attempting to continue." % (self.name, str(type(target))))
except:
raise
elif isinstance(self.type, np.ndarray):
temp = aast.literal_eval(self.tempdata)
if self.inUtil:
target = self.Util
elif self.inBDP:
target = self.BDP
elif self.inAT:
target = self.curAT
else:
target = self.admit
try:
self.setattr(target, self.name, np.array(temp, dtype=object))
except AttributeError:
logging.info("Data member %s is not a member of %s. This may be due to a version mismatch between the data and your software, attempting to continue." % (self.name, str(type(target))))
except:
raise
elif self.inAT and name == self.curAT.show():
self.inAdmit = False
# one last validation run
self.curAT._bdp_in = [None] * len(self.curAT._bdp_in_map)
self.curAT._bdp_out = [None] * len(self.curAT._bdp_out_map)
self.curAT._baseDir = self.basedir
at = copy.deepcopy(self.curAT)
self.AT.append(at)
self.flowmanager[at._taskid] = at
self.curAT = None
self.inAT = False
elif name == bt.ADMIT:
self.inAdmit = False
if not self.dtd.checkAll():
print "Some required nodes missing from admit.xml file, attempting to continue anyway"
elif name == "_keys":
self.inKeys = False
elif name == self.summaryEntryName:
self.summaryData._metadata[self.metadataName].append(copy.deepcopy(self.summaryEntry))
self.summaryEntryName = None
self.inSummaryEntry = False
elif name == self.summaryName:
self.inSummary = False
elif name == self.metadataName:
self.metadataName = None
else:
self.ndarr = []
self.sets = []
self.type = None
self.name = None
self.ndarr = False
