def checkAll(self): """ Method to check the dtd structure to see if all expected nodes were found. Parameters ---------- None Returns ------- Boolean, whether or not all nodes were found """ #pp.pprint(self.entities) for i in self.entities: if not self.entities[i]["found"]: print self.xmlFile logging.info(str(i) + " not found") return False for a in self.entities[i]["attrib"]: if not self.entities[i]["attrib"][a]["found"]: print "2",self.xmlFile logging.info(str(i) + " " + str(a) + " not found") return False return True
def set_mask(self, mask): """ Method to set the mask item Parameters ---------- mask : array like or bool The mask to use, must be equal in dimmension to the spectral axis, or a single boolean value which is applied to all data. Returns ------- None """ tempmask = self._mask if isinstance(mask, ma.masked_array): if len(self._spec) > len(mask) > 1: raise elif len(mask) == 1: self._mask = np.array([mask.data[0]] * len(self._spec)) else: self._mask = np.array(mask.data) elif isinstance(mask, list) or isinstance(mask, np.ndarray): if len(self._spec) > len(mask) > 1: raise elif len(mask) == 1: self._mask = np.array([mask[0]] * len(self._spec)) else: self._mask = np.array(mask) if self.integritycheck(): return logging.warning(" Mask not applied") self._mask = tempmask
def mask_outside(self, limit1, limit2, axis="spec"): """ Method to mask any data less than or greater than the given values. Any axis can be used ("spec", "freq", "chans") as the basis for the masking. Parameters ---------- limit1 : float or int The value below which all data of the given axis will be masked. limit2 : float or int The value above which all data of the given axis will be masked. axis : str The axis which is used to determine the flags. Default: "spec" Returns ------- None """ if not self.integritycheck(): logging.warning(" Masking operation not performed.") return mask1 = getattr(self, "_" + axis, None) < limit1 mask2 = limit2 < getattr(self, "_" + axis, None) mask = np.logical_and(mask1, mask2) self._mask = np.logical_or(self._mask, mask)
def test_warning(self): msg = "unit_test_warning_message" Alogging.warning(msg) found = False r = open(self.logfile, 'r') for line in r.readlines(): if msg in line: if(self.verbose): print "\nFound message >", line found = True r.close() break # since this is the last test case, we now close off the logging Alogging.shutdown() try: if os.path.exists(self.logfile): os.remove(self.logfile) except RuntimeError: pass self.assertTrue(found)
def admit_root(path = None): """ Return the root directory of the ADMIT environment Typically in ADMIT/etc there are files needed by certain functions and other TBD locations. For now we are using getenv(), but in deployment this may not be the case. Parameters ---------- path : string Optional path appended to the ADMIT root Returns ------- String containing the absolute address of the admit root directory, with the optional path appended """ global _admit_root if _admit_root != None: return _admit_root # try the old style developer environment first _admit_root = os.getenv("ADMIT") if _admit_root == None: # try the dumb generic way; is that safe to reference from __file__ ??? _admit_root = version.__file__.rsplit('/',2)[0] if _admit_root[0] != '/': logging.warning("ADMIT_ROOT is a relative address") # @tdodo shouldn't that be a fatal error # print "_ADMIT_ROOT=",_admit_root if path == None: return _admit_root return _admit_root + '/' + path
def mask_ge(self, limit, axis="spec"): """ Method to mask any data less than or equal to the given value. Any axis can be used ("spec", "freq", "chans") as the basis for the masking. Parameters ---------- limit : float or int The value which all data, of the given axis, greater than or equal to, will be masked. axis : str The axis which is used to determine the flags. Default: "spec" Returns ------- None """ if not self.integritycheck(): logging.warning(" Masking operation not performed.") return mask = getattr(self, "_" + axis, None) >= limit self._mask = np.logical_or(self._mask, mask)
def addBDPtoAT(self, bdp): """ Method to add a BDP to an AT. The AT is not specified, but the _taskid attribute of the BDP is used to identify the necessary AT. Parameters ---------- bdp : BDP Any valid BDP, to be added to an existing AT. Returns ------- None """ found = False cp = copy.deepcopy(bdp) # find the AT we need for at in self.tasks: # see if the ID's match if at._taskid == bdp._taskid: found = True # set the base directory of the BDP cp.baseDir(at.baseDir()) # add it to the correct slot at._bdp_out[at._bdp_out_map.index(cp._uid)] = cp break if not found: logging.info("##### Found orphaned BDP with type %s in file %s" % \ (bdp._type, bdp.xmlFile))
def admit_root(path=None): """ Return the root directory of the ADMIT environment Typically in ADMIT/etc there are files needed by certain functions and other TBD locations. For now we are using getenv(), but in deployment this may not be the case. Parameters ---------- path : string Optional path appended to the ADMIT root Returns ------- String containing the absolute address of the admit root directory, with the optional path appended """ global _admit_root if _admit_root != None: return _admit_root # try the old style developer environment first _admit_root = os.getenv("ADMIT") if _admit_root == None: # try the dumb generic way; is that safe to reference from __file__ ??? _admit_root = version.__file__.rsplit('/', 2)[0] if _admit_root[0] != '/': logging.warning("ADMIT_ROOT is a relative address") # @tdodo shouldn't that be a fatal error # print "_ADMIT_ROOT=", _admit_root if path == None: return _admit_root return _admit_root + '/' + path
def end(self): t0 = self.init t1 = self.time() dt = t1 - t0 if self.report: logging.timing("%s END " % self.label + str(dt)) return dt
def get_mem(self): """ Read memory usage info from /proc/pid/status Return Virtual and Resident memory size in MBytes. """ global ostype if ostype == None: ostype = os.uname()[0].lower() logging.info("OSTYPE: %s" % ostype) scale = {'MB': 1024.0} lines = [] try: if ostype == 'linux': proc_status = '/proc/%d/status' % os.getpid() # linux only # open pseudo file /proc/<pid>/status t = open(proc_status) # get value from line e.g. 'VmRSS: 9999 kB\n' for it in t.readlines(): if 'VmSize' in it or 'VmRSS' in it : lines.append(it) t.close() else: proc = subprocess.Popen(['ps','-o', 'rss', '-o', 'vsz', '-o','pid', '-p',str(os.getpid())],stdout=subprocess.PIPE) proc_output = proc.communicate()[0].split('\n') proc_output_memory = proc_output[1] proc_output_memory = proc_output_memory.split() phys_mem = int(proc_output_memory[0])/1204 # to MB virtual_mem = int(proc_output_memory[1])/1024 except (IOError, OSError): if self.report: logging.timing(self.label + " Error: cannot read memory usage information.") return np.array([]) # parse the two lines mem = {} if(ostype != 'darwin'): for line in lines: words = line.strip().split() #print words[0], '===', words[1], '===', words[2] # get rid of the tailing ':' key = words[0][:-1] # convert from KB to MB scaled = float(words[1]) / scale['MB'] mem[key] = scaled else: mem['VmSize'] = virtual_mem mem['VmRSS'] = phys_mem return np.array([mem['VmSize'], mem['VmRSS']])
def __init__(self, upper=True): self.version = "27-apr-2016" if have_ADMIT: self.table = utils.admit_root() + "/etc/vlsr.tab" self.cat = read_vlsr(self.table,upper) logging.debug("VLSR: %s, found %d entries" % (self.table,len(self.cat))) else: logging.warning("VLSR: Warning, no ADMIT, empty catalogue") self.cat = {}
def __init__(self, label=".", report=True): self.start = self.time() self.init = self.start self.label = label self.report = report self.dtimes = [] dt = self.init - self.init if self.report: logging.timing("%s ADMIT " % self.label + str(self.start)) logging.timing("%s BEGIN " % self.label + str(dt))
def tab_to_slit(xym, clip=0.0, gamma=1.0): """take all values from a map over clip, compute best slit for PV Slice """ x = xym[0] # maxposx y = xym[1] # maxposy m = xym[2] # max logging.debug("CLIP %g" % clip) slit = convert_to_slit(m,x,y,0,0,gamma,expand=2.0) return (slit,clip)
def error(self, ex): """ Method called when an error is encountered Parameters ---------- ex : exception The error that was encountered Returns ------- None """ logging.warning("Recoverable error encountered: %s" % (ex.message))
def warning(self, ex): """ Method called when the parser issues a warning Parameters ---------- ex : exception The warning to be issued. Returns ------- None """ logging.warning("Warning: %s" % (ex.message))
def peakstats(image, freq, sigma, nsigma, minchan, maxgap, psample, peakfit = False): """ Go through a cube and find peaks in the spectral dimension It will gather a table of <peak>,<freq>,<sigma> which can be optionally used for plotting """ if psample < 0: return cutoff = nsigma * sigma madata = casautil.getdata(image) data = madata.data shape = data.shape logging.debug("peakstats: shape=%s cutoff=%g" % (str(shape),cutoff)) #print "DATA SHAPE:",shape #print "cutoff=",cutoff nx = shape[0] ny = shape[1] nz = shape[2] chan = np.arange(nz) # prepare the segment finder # we now have an array data[nx,ny,nz] sum = 0.0 pval = [] mval = [] wval = [] for x in range(0,nx,psample): for y in range(0,ny,psample): s0 = data[x,y,:] spec = ma.masked_invalid(s0) sum += spec.sum() # using abs=True is a bit counter intuitive, but a patch to deal with the confusion in # ADMITSegmentFinder w.r.t abs usage asf = ADMITSegmentFinder(pmin=nsigma, minchan=minchan, maxgap=maxgap, freq=freq, spec=spec, abs=True) #asf = ADMITSegmentFinder(pmin=nsigma, minchan=minchan, maxgap=maxgap, freq=freq, spec=spec, abs=False) f = asf.line_segments(spec, nsigma*sigma) for s in f: if False: for i in range(s[0],s[1]+1): print "# ",x,y,i,spec[i] ## area preserving and peak are correlated, 18% difference ## fitgauss1Dm was about 5" ## with fitgauss1D was about 30", and still bad fits par = utils.fitgauss1Dm(chan[s[0]:s[1]+1], spec[s[0]:s[1]+1], True) # peak from max #par = utils.fitgauss1Dm(chan[s[0]:s[1]+1], spec[s[0]:s[1]+1], False) # peak from area preserving if peakfit: (par,cov) = utils.fitgauss1D (chan[s[0]:s[1]+1], spec[s[0]:s[1]+1],par) #print "FIND: ",x,y,s,cutoff,0.0,0.0,par[0],par[1],par[2],s[1]-s[0]+1 pval.append(par[0]) mval.append(par[1]) wval.append(par[2]) #print "SUM:",sum return (np.array(pval),np.array(mval),np.array(wval))
def fatalError(self, ex): """ Method called when a fata error is encountered Parameters ---------- ex : exception The error encountered Returns ------- None """ logging.error("Fatal error encountered.") raise ex
def fitgauss1D(x, y, par=None, width=-1.0): """ Method for fitting a 1D gaussian to a spectral line Parameters ---------- x: array like The x co-ordinates of the spectrum, note the center of the spectral line should be near 0.0 if possible y: array like The y co-ordinates (intensity) of the spectrum par: array like The initial guesses for the fit parameters, the fitter works best if the center parameter is near 0.0 3 parameters: PeakY, CenterX, FWHM. width: float If positive, this is the assumed width (or step) in the x array, which is needed if only 1 point is given. Otherwise ignored. Returns ------- A tuple containing the best fit parameters (as a list) and the covariance of the parameters (also as a list) """ if len(x) == 3: logging.info( "Gaussian fit attempted with only three points, look at the covariance for goodness of fit." ) # if there are too few points to fit then just conserve the are of the channels to calculate the # parameters if len(x) < 3: logging.info( "Gaussian fit attempted with fewer than three points (%d). Using conservation of area method to determine parameters." % len(x)) params = fitgauss1Dm(x, y, dx=width) covar = [1000.] * len(params) else: try: params, covar = curve_fit(gaussian1D, x, y, p0=par) # if the covariance cannot be determined, just return the initial values except RuntimeError, e: if "Optimal" in str(e): params = par covar = [0] * len(par) # otherwise re-raise the exception else: raise
def test_debug(self): msg = "unit_test_debug_message" Alogging.debug(msg) found = False r = open(self.logfile, 'r') for line in r.readlines(): if msg in line: if(self.verbose): print "\nFound message > ", line found = True r.close() break self.assertTrue(found)
def tag(self, mytag): t0 = self.start t1 = self.time() dt = t1 - t0 # get memory usage (Virtual and Resident) info mem = self.get_mem() if mem.size != 0: dt = np.append(dt, mem) self.dtimes.append((mytag, dt)) self.start = t1 if self.report: logging.timing("%s " % self.label + mytag + " " + str(dt)) return dt
def tag(self, mytag): t0 = self.start t1 = self.time() dt = t1 - t0 # get memory usage (Virtual and Resident) info mem = self.get_mem() if mem.size != 0 : dt = np.append(dt, mem) self.dtimes.append((mytag, dt)) self.start = t1 if self.report: logging.timing("%s " % self.label + mytag + " " + str(dt)) return dt
def test_reportKeywords(self): kw = {"input": "helloWorld", "list" : [1,2,3,4]} Alogging.reportKeywords(kw) found = [] r = open(self.logfile, 'r') for line in r.readlines(): for k, v in kw.iteritems(): if k in line and str(v) in line: if(self.verbose): print "\nFound message >", line found.append(True) r.close() self.assertTrue(len(found) == 2)
def fitgauss1D(x, y, par=None, width=-1.0): """ Method for fitting a 1D gaussian to a spectral line Parameters ---------- x: array like The x co-ordinates of the spectrum, note the center of the spectral line should be near 0.0 if possible y: array like The y co-ordinates (intensity) of the spectrum par: array like The initial guesses for the fit parameters, the fitter works best if the center parameter is near 0.0 3 parameters: PeakY, CenterX, FWHM. width: float If positive, this is the assumed width (or step) in the x array, which is needed if only 1 point is given. Otherwise ignored. Returns ------- A tuple containing the best fit parameters (as a list) and the covariance of the parameters (also as a list) """ if len(x) == 3: logging.info("Gaussian fit attempted with only three points, look at the covariance for goodness of fit.") # if there are too few points to fit then just conserve the are of the channels to calculate the # parameters if len(x) < 3: logging.info("Gaussian fit attempted with fewer than three points (%d). Using conservation of area method to determine parameters." % len(x)) params = fitgauss1Dm(x,y,dx=width) covar = [1000.] * len(params) else: try: params, covar = curve_fit(gaussian1D, x, y, p0=par) # if the covariance cannot be determined, just return the initial values except RuntimeError, e: if "Optimal" in str(e): params = par covar = [0] * len(par) # otherwise re-raise the exception else: raise
def test_effectiveLevel(self): msg = "unit_test_levels_message" # check that the logging level is what is expected level = Alogging.getEffectiveLevel() self.assertTrue(level == self.level) # set the level to a new value and check again Alogging.setLevel(50) level = Alogging.getEffectiveLevel() self.assertTrue(level == 50) # log an info message which is below the logging level, this message should not appear # in the logs Alogging.info(msg) found = False r = open(self.logfile, 'r') for line in r.readlines(): if msg in line: if(self.verbose): print "\nFound message >", line found = True break r.close() self.assertFalse(found) Alogging.setLevel(self.level) # reset the logging level msg += "2" # log an info message, which is now above the logging level, this message should appear # in the logs Alogging.info(msg) found = False r = open(self.logfile, 'r') for line in r.readlines(): if msg in line: if(self.verbose): print "\nFound message >", line found = True r.close() break self.assertTrue(found)
def mask_invalid(self): """ Method to mask all invalid spectral data. Invalid values are: NaN, Inf, -Inf Parameters ---------- None Returns ------- None """ if not self.integritycheck(): logging.warning(" Masking operation not performed.") return mask = np.isfinite(self._spec) self._mask = np.logical_or(self._mask, np.logical_not(mask))
def fitgauss1Dm(xdat, ydat, usePeak=False, dx=-1.0): """ gaussfit helper function this will get a reasonable gauss even if you only have 2 points assumes evenly spaced data in xdat, so it can extract a width. It can be used like fitgauss1D(), but uses the first three moments of the distribution to "match" that of a gauss. area preserving if you wish. If you set usePeak, it will pick the highest value in your data. Warning: if you must fit negative profiles, be sure to rescale before you come into this routine. """ if len(xdat) == 1: # special case, it will need dx if dx < 0.0: logging.critical( "Cannot determine gaussian of delta function if width not given" ) raise return (ydat[0], xdat[0], dx) sum0 = sum1 = sum2 = peak = 0.0 # the mean is given as the weighted mean of x for x, y in zip(xdat, ydat): if y > peak: peak = y sum0 = sum0 + y sum1 = sum1 + y * x xmean = sum1 / sum0 # re-center the data for a moment-2 calculation # @todo pos/neg xdat0 = xdat - xmean for x, y in zip(xdat0, ydat): sum2 = sum2 + y * x * x sigma = math.sqrt(abs(sum2 / sum0)) # equate the area under the histogram with the area under a gauss # to get the peak of the "matched" gauss dx = abs(xdat[1] - xdat[0]) # @todo use optional "dx=None" ? if not usePeak: # pick the area preserving one, vs. the "real" peak # The area preserving one seems to be about 18% higher # but with a tight correllation peak = (sum0 * dx) / math.sqrt(2 * sigma * math.pi) fwhm = 2.35482 * sigma return (peak, xmean, fwhm)
def fitgauss1Dm(xdat, ydat, usePeak = False, dx = -1.0): """ gaussfit helper function this will get a reasonable gauss even if you only have 2 points assumes evenly spaced data in xdat, so it can extract a width. It can be used like fitgauss1D(), but uses the first three moments of the distribution to "match" that of a gauss. area preserving if you wish. If you set usePeak, it will pick the highest value in your data. Warning: if you must fit negative profiles, be sure to rescale before you come into this routine. """ if len(xdat) == 1: # special case, it will need dx if dx < 0.0: logging.critical("Cannot determine gaussian of delta function if width not given") raise return (ydat[0], xdat[0], dx) sum0 = sum1 = sum2 = peak = 0.0 # the mean is given as the weighted mean of x for x,y in zip(xdat,ydat): if y>peak: peak = y sum0 = sum0 + y sum1 = sum1 + y*x xmean = sum1/sum0 # re-center the data for a moment-2 calculation # @todo pos/neg xdat0 = xdat - xmean for x,y in zip(xdat0,ydat): sum2 = sum2 + y*x*x sigma = math.sqrt(abs(sum2/sum0)) # equate the area under the histogram with the area under a gauss # to get the peak of the "matched" gauss dx = abs(xdat[1]-xdat[0]) # @todo use optional "dx=None" ? if not usePeak: # pick the area preserving one, vs. the "real" peak # The area preserving one seems to be about 18% higher # but with a tight correllation peak = (sum0 * dx) / math.sqrt(2*sigma*math.pi) fwhm = 2.35482 * sigma return (peak, xmean, fwhm)
def setUp(self): self.verbose = False self.testName = "Utility AdmitLogging Class Unit Test" self.logfile = '/tmp/logging_unit_test_%s.log' % os.getpid() self.level = 1 # only need to initialize the logger once since it is a static class if not TestAdmitLogging.setup: Alogging.init(name="test", logfile=self.logfile, level=Alogging.DEBUG) Alogging.addLevelName(15, "TIMING") Alogging.addLevelName(16, "REGRSSION") Alogging.setLevel(self.level) TestAdmitLogging.setup = True
def checkAttribute(self, name, attrib, value=None): """ Method to check an attribute for validity. Validity includes correct name and data type. Parameters ---------- name : str The name of the node being checked attrib : str The attribute of the node being checked, if any. value : str The type of the attribute being checked (e.g. bt.INT) Default: None """ # check an attribute for validity try: if not value in self.entities[name]["attrib"][attrib]["values"] \ and not "ANY" in self.entities[name]["attrib"][attrib]["values"]: raise Exception("DTDParser.checkAttributes: Value %s for attribute %s is not a valid entry (file %s)" % (value, name, self.xmlFile)) self.entities[name]["attrib"][attrib]["found"] = True except KeyError: logging.info("Attribute %s not listed in DTD, malformed xml detected (%s)" % (attrib, self.xmlFile)) logging.info("Inconsistency between dtd and xml detected, continuing") except: logging.info("Unknown error encountered while parsing attribute %s (%s)" % (attrib, self.xmlFile)) raise
def integritycheck(self): """ Method to check that all axes are the same length. Axes that have no data are ignored. Parameters ---------- None Returns ------- Bool, True if all match, False if there is an inconsistency """ if self._spec is not None: ls = len(self._spec) else: ls = 0 if self._freq is not None: lf = len(self._freq) else: lf = ls if self._chans is not None: lc = len(self._chans) else: lc = ls if self._mask is not None: lm = len(self._mask) else: lm = ls if self._contin is not None: lco = len(self._contin) else: lco = ls if ls == lf == lc == lm == lco: return True logging.warning( "Inconsistent axes: spectrum: %i, freq: %i, chans: %i, mask: %i, contin: %i" % (ls, lf, lc, lm, lco)) return False
def fix_invalid(self, mask_value=0.0): """ Method to replace invalid spectral data with a specific value and mask it. Invalid values are: NaN, Inf, -Inf Parameters ---------- mask_value : float The value to replace the invalid data with. Default: 0.0 Returns ------- None """ if not self.integritycheck(): logging.warning(" Masking operation not performed.") return mask = np.isfinite(self._spec) for i in range(len(mask)): if not mask[i]: self._spec[i] = mask_value self._mask = np.logical_or(self._mask, np.logical_not(mask))
def run(self): """ running the File_AT task """ # grab and check essential keywords filename = self.getkey('file') logging.info("file=%s" % filename) if len(filename) == 0: raise Exception,'File_AT: no file= given' exist = self.getkey('exist') if exist: # logging.warning("no checking now") # self._bdp_in[0].checkfiles() # create the BDP bdp1 = File_BDP(filename) bdp1.filename = filename self.addoutput(bdp1) # touch the file if desired if self.getkey('touch'): bdp1.touch()
def run(self): """ running the File_AT task """ # grab and check essential keywords filename = self.getkey('file') logging.info("file=%s" % filename) if len(filename) == 0: raise Exception, 'File_AT: no file= given' exist = self.getkey('exist') if exist: # logging.warning("no checking now") # self._bdp_in[0].checkfiles() # create the BDP bdp1 = File_BDP(filename) bdp1.filename = filename self.addoutput(bdp1) # touch the file if desired if self.getkey('touch'): bdp1.touch()
def integritycheck(self): """ Method to check that all axes are the same length. Axes that have no data are ignored. Parameters ---------- None Returns ------- Bool, True if all match, False if there is an inconsistency """ if self._spec is not None: ls = len(self._spec) else: ls = 0 if self._freq is not None: lf = len(self._freq) else: lf = ls if self._chans is not None: lc = len(self._chans) else: lc = ls if self._mask is not None: lm = len(self._mask) else: lm = ls if self._contin is not None: lco = len(self._contin) else: lco = ls if ls == lf == lc == lm == lco: return True logging.warning("Inconsistent axes: spectrum: %i, freq: %i, chans: %i, mask: %i, contin: %i" % (ls, lf, lc, lm, lco)) return False
def parse_response(self): """ Parse a response (of type urllib2.urlopen) into a list of dictionaries. Parameters ---------- None Returns ------- A dictonary of the response """ result = [] response = self.response try: csvstring = response.read() tablelist = csvstring.split('\n') header = tablelist[0] headerlist = header.split(':') headerlist = clean_column_headings(headerlist, renaming_dict=column_headings_map) #Populate list of dictionaries for row in tablelist[1:-1]: rowlist = row.split(':') rowdict = {} for i in range(0, len(rowlist)): rowdict[headerlist[i]] = rowlist[i] result.append(rowdict) except: logging.warning("Problem parsing result") self.result = result return result
def characters(self, ch): """ Method called whenever characters are detected in an xml node This method does some dtd validation. This method is only called by the SAX parser iteself. Parameters ---------- ch : unicode characters Returns ------- None """ target = None char = str(ch).strip() if char.isspace() or not char: return # determine which class the data are getting writtrn to if self.inUtil: target = self.Util elif self.inBDP: target = self.BDP elif self.inAT: target = self.curAT elif self.inSummaryEntry: target = self.summaryEntry elif self.inSummary: target = self.summaryData else: target = self.admit # a list or dictionary has to be decoded if isinstance(self.type, list) or isinstance(self.type, dict) \ or isinstance(self.type, tuple) or isinstance(self.type, set) \ or isinstance(self.type, np.ndarray) or isinstance(self.type, str): if self.inflow: self.flowdata += char else: self.tempdata += char else: # check the version if self.name == "_version": ver = self.getattr(target, self.name) vercheck = utils.compareversions(ver, str(char)) if vercheck < 0: # newer read in logging.warning("Version mismatch for %s, data are a newer version than current software, attempting to continue." % target.getkey("_type")) elif vercheck > 0: # older read in logging.warning("Version mismatch for %s, data are an older version than current software, attempting to continue." % target.getkey("_type")) else: try: self.setattr(target, self.name, self.getData(char)) except AttributeError: logging.info("Data member %s is not a member of %s. This may be due to a version mismatch between the data and your software, attempting to continue." % (self.name, str(type(target)))) except: raise del ch
def check(self, name, attrib=None, value=None): """ Method to check a node for validity. Validity includes correct name and data type. Parameters ---------- name : str The name of the node being checked attrib : str The attribute of the node being checked, if any. Default: None value : str The type of the attribute being checked (e.g. bt.INT) Default: None """ # check a node for validity try: # note that the node has been found self.entities[name]["found"] = True # if there is an attribute specified then check it too # if the attribute was not expected just print a note to the screen if attrib is not None: try: if not value in self.entities[name]["attrib"][attrib]["values"] \ and not "ANY" in self.entities[name]["attrib"][attrib]["values"]: raise Exception("DTDParser.check: Value %s for attribute %s is not a valid entry (attribute = %s) (file %s)" % (value, name, attrib, self.xmlFile)) self.entities[name]["attrib"][attrib]["found"] = True except KeyError: logging.info("Attribute %s for %s not listed in DTD, malformed xml detected (%s)" % (attrib, name, self.xmlFile)) logging.info("Inconsistency between dtd and xml detected, continuing") except: logging.info("Unknown error encountered while parsing attribute %s for %s (%s)" % (attrib, name, self.xmlFile)) raise except KeyError: logging.info("Data member %s is not a member of the dtd, xml inconsistent with definition (%s)" % (name, self.xmlFile)) except: raise
def run(self): """ The run method creates the BDP Parameters ---------- None Returns ------- None """ dt = utils.Dtime("Export") # tagging time basename = self.getkey("basename") nbdp = len(self._bdp_in) logging.info("Found %d input BDPs" % nbdp) if nbdp > 1: logging.info("Only dealing with 1 BDP now") b1 = self._bdp_in[0] # image/cube infile = b1.getimagefile(bt.CASA) # ADMIT filename of the image (cube) if len(basename) == 0: fitsname = self.mkext( infile, 'fits' ) # morph to the new output name with replaced extension ' image_out = self.dir(fitsname) # absolute filename else: if basename[0:2] == './' or basename[0] == '/': image_out = basename + ".fits" else: image_out = self.dir(basename + ".fits") dt.tag("start") logging.info("Writing FITS %s" % image_out) # @todo check self.dir(image_out) casa.exportfits(self.dir(infile), image_out, overwrite=True) dt.tag("done") dt.end()
def run(self): """ The run method creates the BDP Parameters ---------- None Returns ------- None """ dt = utils.Dtime("Export") # tagging time basename = self.getkey("basename") nbdp = len(self._bdp_in) logging.info("Found %d input BDPs" % nbdp) if nbdp > 1: logging.info("Only dealing with 1 BDP now") b1 = self._bdp_in[0] # image/cube infile = b1.getimagefile(bt.CASA) # ADMIT filename of the image (cube) if len(basename) == 0: fitsname = self.mkext(infile,'fits') # morph to the new output name with replaced extension ' image_out = self.dir(fitsname) # absolute filename else: if basename[0:2] == './' or basename[0] == '/': image_out = basename + ".fits" else: image_out = self.dir(basename + ".fits") dt.tag("start") logging.info("Writing FITS %s" % image_out) # @todo check self.dir(image_out) casa.exportfits(self.dir(infile), image_out, overwrite=True) dt.tag("done") dt.end()
def run(self): """ The run method, creates the slices, regrids if requested, and creates the BDP(s) Parameters ---------- None Returns ------- None """ dt = utils.Dtime("LineCube") self._summary = {} # look for an input noise level, either through keyword or input # CubeStats BDP or calculate it if needed pad = self.getkey("pad") equalize = self.getkey("equalize") minchan = 0 linelist = self._bdp_in[1] if linelist == None or len(linelist) == 0: logging.info("No lines found in input LineList_BDP, exiting.") return spw = self._bdp_in[0] # get the columns from the table cols = linelist.table.getHeader() # get the casa image imagename = spw.getimagefile(bt.CASA) imh = imhead(self.dir(imagename), mode='list') # set the overall parameters for imsubimage args = {"imagename" : self.dir(imagename), "overwrite" : True} dt.tag("start") if pad != 0: nchan = imh['shape'][2] dt.tag("pad") # if equal size cubes are requested, this will honor the requested pad if equalize: start = linelist.table.getColumnByName("startchan") end = linelist.table.getColumnByName("endchan") # look for the widest line for i in range(len(start)): diff = end[i] - start[i] + 1 minchan = max(minchan , diff + (pad * 2)) dt.tag("equalize") # get all of the rows in the table rows = linelist.getall() delrow = set() procblend = [0] # search through looking for blended lines, leave only the strongest from each blend # in the list for i, row in enumerate(rows): if row.blend in procblend: continue strongest = -100. index = -1 indexes = [] blend = row.blend for j in range(i, len(rows)): if rows[j].blend != blend: continue indexes.append(j) if rows[j].linestrength > strongest: strongest = rows[j].linestrength index = j indexes.remove(index) delrow = delrow | set(indexes) procblend.append(blend) dr = list(delrow) dr.sort() dr.reverse() for row in dr: del rows[row] # check on duplicate UID's, since those are the directory names here uid1 = [] for row in rows: uid1.append(row.getkey("uid")) uid2 = set(uid1) if len(uid1) != len(uid2): print "LineList:",uid1 logging.warning("There are duplicate names in the LineList") #raise Exception,"There are duplicate names in the LineList" # Create Summary table lc_description = admit.util.Table() lc_description.columns = ["Line Name","Start Channel","End Channel","Output Cube"] lc_description.units = ["","int","int",""] lc_description.description = "Parameters of Line Cubes" # loop over all entries in the line list rdata = [] for row in rows: uid = row.getkey("uid") cdir = self.mkext(imagename,uid) self.mkdir(cdir) basefl = uid lcd = [basefl] outfl = cdir + os.sep + "lc.im" args["outfile"] = self.dir(outfl) start = row.getkey("startchan") end = row.getkey("endchan") diff = end - start + 1 startch = 0 if diff < minchan: add = int(math.ceil(float(minchan - diff) / 2.0)) start -= add end += add startch += add if start < 0: logging.info("%s is too close to the edge to encompass with the " + "requested channels, start=%d resetting to 0" % (uid, start)) startch += abs(start) start = 0 if end >= nchan: logging.info("%s is too close to the edge to encompass with the " + "requested channels, end=%d resetting to %d" % (uid, end, nchan - 1)) end = nchan - 1 #print "\n\nDIFF ",startch,"\n\n" if pad > 0 and not equalize: start -= pad end += pad if start < 0: logging.warning("pad=%d too large, start=%d resetting to 0" % (pad, start)) startch += abs(start) start = 0 else: startch += pad if end >= nchan: logging.warning("pad=%d too large, end=%d resetting to %d" % (pad, end, nchan - 1)) end = nchan - 1 elif pad < 0 and not equalize: mid = (start + end) / 2 start = mid + pad / 2 end = mid - pad / 2 - 1 if start < 0: logging.warning("pad=%d too large, start=%d resetting to 0" % (pad, start)) startch += abs(start) start = 0 else: startch += abs(start) if end >= nchan: logging.warning("pad=%d too large, end=%d resetting to %d" % (pad, end, nchan - 1)) end = nchan - 1 endch = startch + diff args["chans"] = "%i~%i" % (start, end) rdata.append(start) rdata.append(end) # for the summmary, which will be a table of # Line name, start channel, end channel, output image lc_description.addRow([basefl, start, end, outfl]) # create the slices imsubimage(**args) line = row.converttoline() # set the restfrequency ouf the output cube imhead(imagename=args["outfile"], mode="put", hdkey="restfreq", hdvalue="%fGHz" % (row.getkey("frequency"))) # set up the output BDP images = {bt.CASA : outfl} casaimage = Image(images=images) # note that Summary.getLineFluxes() implicitly relies on the BDP out order # being the same order as in the line list table. If this is ever not # true, then Summary.getLineFluxes mismatch BDPs and flux values. #self.addoutput(LineCube_BDP(xmlFile=cdir + os.sep + basefl + ".lc", self.addoutput(LineCube_BDP(xmlFile=outfl, image=casaimage, line=line, linechans="%i~%i" % (startch, endch))) dt.tag("trans-%s" % cdir) logging.regression("LC: %s" % str(rdata)) taskargs = "pad=%s equalize=%s" % (pad, equalize) self._summary["linecube"] = SummaryEntry(lc_description.serialize(), "LineCube_AT", self.id(True), taskargs) dt.tag("done") dt.end()
def endElement(self, name): """ Method called whenever the end of an xml element is reached. This method is only called by the SAX parser iteself. Parameters ---------- name : str The name of the node that just ended Returns ------- None """ # reset the tracking stuff, add BDP's to AT's, AT's to the flowmanager # reconstruct any nodes that spanned multiple lines if name == self.utilName: # add the utility classes to the appropriate parent class # Images always get added to MultiImages if self.inMulti: self.MultiImage.addimage(copy.deepcopy(self.Util), self.Util.name) elif self.inBDP: setattr(self.BDP, self.utilName, copy.deepcopy(self.Util)) elif self.inAT: setattr(self.curAT, self.utilName, copy.deepcopy(self.Util)) self.inUtil = False self.utilName = "" elif name == self.multiName: if self.inBDP: setattr(self.BDP, self.multiName, copy.deepcopy(self.MultiImage)) elif self.inAT: setattr(self.curAT, self.multiName, copy.deepcopy(self.MultiImage)) self.multiImageName = "" self.inMulti = False self.inUtil = False elif name == bt.BDP: # one last validation run self.BDP._baseDir = self.basedir if not self.dtd.checkAll(): logging.info("Some required nodes missing from xml file, attempting to continue anyway.") elif name == bt.FLOWMANAGER: temp = aast.literal_eval(self.flowdata) for key in ["depsmap", "varimap"]: if key in temp: temp[key] = eval(temp[key]) self.flowmanager = fm.FlowManager(**temp) self.inflow = False elif isinstance(self.type, str): if self.inUtil: target = self.Util elif self.inBDP: target = self.BDP elif self.inAT: target = self.curAT elif self.inSummaryEntry: target = self.summaryEntry elif name == "projmanager": target = self else: target = self.admit self.setattr(target, name, self.tempdata) self.tempdata = "" elif isinstance(self.type, list) or isinstance(self.type, dict) \ or isinstance(self.type, tuple) or isinstance(self.type, set): temp = aast.literal_eval(self.tempdata) if self.inUtil: target = self.Util elif self.inBDP: target = self.BDP elif self.inAT: target = self.curAT elif self.inSummaryEntry: target = self.summaryEntry elif name == "projmanager": target = self else: target = self.admit for i in self.ndarr: temp[i] = np.array(temp[i], dtype=object) for i in self.sets: temp[i] = set(temp[i]) if isinstance(self.type, tuple): temp = tuple(temp) elif isinstance(self.type, set): temp = set(temp) try: self.setattr(target, name, temp) except AttributeError: logging.info("Data member %s is not a member of %s. This may be due to a version mismatch between the data and your software, attempting to continue." % (self.name, str(type(target)))) except: raise elif isinstance(self.type, np.ndarray): temp = aast.literal_eval(self.tempdata) if self.inUtil: target = self.Util elif self.inBDP: target = self.BDP elif self.inAT: target = self.curAT else: target = self.admit try: self.setattr(target, self.name, np.array(temp, dtype=object)) except AttributeError: logging.info("Data member %s is not a member of %s. This may be due to a version mismatch between the data and your software, attempting to continue." % (self.name, str(type(target)))) except: raise elif self.inAT and name == self.curAT.show(): self.inAdmit = False # one last validation run self.curAT._bdp_in = [None] * len(self.curAT._bdp_in_map) self.curAT._bdp_out = [None] * len(self.curAT._bdp_out_map) self.curAT._baseDir = self.basedir at = copy.deepcopy(self.curAT) self.AT.append(at) self.flowmanager[at._taskid] = at self.curAT = None self.inAT = False elif name == bt.ADMIT: self.inAdmit = False if not self.dtd.checkAll(): print "Some required nodes missing from admit.xml file, attempting to continue anyway" elif name == "_keys": self.inKeys = False elif name == self.summaryEntryName: self.summaryData._metadata[self.metadataName].append(copy.deepcopy(self.summaryEntry)) self.summaryEntryName = None self.inSummaryEntry = False elif name == self.summaryName: self.inSummary = False elif name == self.metadataName: self.metadataName = None else: self.ndarr = [] self.sets = [] self.type = None self.name = None self.ndarr = False