def getData(fname): """Extract data from mda-ASCII file and distribute into Pixel objects Returns: XAS scan data in a detector 'object' (variable "det") energy axis, transmission data array and detector filled with fluo data (detector object simply full of '0' values if no fluorescence data available) Transmission data in "trans" comprises encoder_E, I0/1/2, sample_time, encoder_angle !! I0/1/2 are already normalised to t !! """ mda = readMDA.readMDA(fname, verbose=True) scanData = mda[1] scanSize = scanData.npts try: detectorData = mda[2] detSize = detectorData.npts except: detSize = 0 det = makeDet(detSize, scanSize) # read transmission data pvColumnNames = ['EncEnergy:ActPos', 'scaler1:S2C', 'scaler1:S3C', 'scaler1:S4C', 'scaler1.T', 'EncAngle:ActPos'] trans = np.empty((len(pvColumnNames), scanSize)) for series in scanData.d: try: tag = ':'.join(series.name.split(':')[1:]) # use the PV part after the IOC id trans[pvColumnNames.index(tag)] = series.data except: pass ts = trans[pvColumnNames.index('scaler1.T')] # get the sample time "t_s" e = trans[pvColumnNames.index('EncEnergy:ActPos')] * 1000.0 # Energy axis (in eV !!) # normalise I0, I1, I2 to sample_time ts (use string "scaler1:" as identifier) for i, name in enumerate(pvColumnNames): if name.startswith('scaler1:'): trans[i] = trans[i] / ts # read fluorescence data (if it exists) if detSize != 0: print 'reading fluorescence data...' for k in range(scanSize): for i in np.arange(detSize): det[i].fpeaks[k] = detectorData.d[0].data[k][i] det[i].speaks[k] = detectorData.d[1].data[k][i] det[i].roi[k] = detectorData.d[2].data[k][i] for i in np.arange(detSize): det[i].NormT(ts) # ("ts" = sampling time per data point t_s) ## call dead time correction later; should be performed only on ## good spectra ("goodPixels") as bad spectra can contain zeros which ## this function would divide by ## call is via function: detDeadCorr(det, goodPixels) return e, trans, det
def extract_angles(fn_string, fn_range): """ Convenience function for extracting the angles from a bunch of mda files Inputs ------ fn_string: the path to the mda files with {:04d} (or similar) for specifying the number. fn_range: a two element tuple specifying the dtart and end numbers Outputs ------- A pretty list of mda number and angle which can be cut and pasted into a google doc. """ print "This process takes so long because the entire mda file is parsed despite\nneeding but a single element.\nC'est la vie.\n\n" l = [] for i in range(fn_range[0], fn_range[1]): print 'Extracting angles from mda: {:04d}'.format(i) mda = readMDA(fn_string.format(i), verbose=0) l.append((i, mda[0]['2xfm:userStringCalc10.DD'][2])) print 'MDA\t\tAngle (degrees)' pstr = '\t{:04d}\t\t{:3.2f}' for element in l: print pstr.format(element[0], float(element[1]))
def convertMDA(self): mdaFile = '%s%s' % (self.mda_path,self.scanFileNamePV.get(as_string=True)) print mdaFile try: mda = readMDA(mdaFile, verbose=False) except Exception: return data = mda[-1] results = [p.data for p in data.p] results.extend([d.data for d in data.d]) results = zip(*results) names = [p.name for p in data.p] names.extend([d.name for d in data.d]) datFileName = '%s%s.csv' % (self.dataFullPath, os.path.splitext(os.path.basename(mdaFile))[0]) print datFileName with open(datFileName,'w') as f: print >> f, ', '.join('%s' % name for name in names) for line in results: print >> f, ', '.join('%s' % data for data in line) epn = self.dataFullPath.split('/')[-3] try: self.redis.rpush('MDA:%s' % (epn,),datFileName) self.redis.publish('MDA:NewFile',datFileName) except redis.ConnectionError: print 'Error connecting to redis database'
def panMDA(file=''): "panMDA(file='') - display all 2D images from an MDA file in a scrolled window" if file == '': file ='/home/beams/CHA/data/xxx/cha_0001.mda' d = readMDA(file) pal = readPalette() det2D(d[2].d[0:d[2].nd],scale=(2,2),columns=5,file=file,pal=pal) return d
def panMDA(file=''): "panMDA(file='') - display all 2D images from an MDA file in a scrolled window" if file == '': file = '/home/beams/CHA/data/xxx/cha_0001.mda' d = readMDA(file) pal = readPalette() det2D(d[2].d[0:d[2].nd], scale=(2, 2), columns=5, file=file, pal=pal) return d
def readDir(s): dd = [] files = glob(s) i = 0 for file in files: dd.append(readMDA.readMDA(file, verbose=0, maxdim=0)) print "file:%-15s rank:%d %s" % (dd[i][0]['filename'], dd[i][0]['rank'], str(dd[i][0]['dimensions'])) i = i+1 return dd
def readDir(s): dd = [] files = glob(s) i = 0 for file in files: dd.append(readMDA.readMDA(file, verbose=0, maxdim=0)) print "file:%-15s rank:%d %s" % (dd[i][0]['filename'], dd[i][0]['rank'], str(dd[i][0]['dimensions'])) i = i + 1 return dd
def getAllExtraPVs(fname): """Return a dictionary of all "Extra" PVs, indexed by the part of the PV name following the colon, e.g. if fname contains the PV SR12ID01DET01:mca1.R0LO with a value ("long", "", [1200]), the returned dict will contain a key:value entry 'mca1.R0LO':("long", "", [1200]), where the value is the triple obtained from readMDA Arguments: fname - mda filename Returns: dict e.g. {'mca1.R0LO':("long", "", [1000]), 'mca1.R0HI':("long", "", [1200]), ...} """ # get the path to the netCDF files from the mda file mda = readMDA.readMDA(fname, verbose=False) extra_pvs = mda[0] # get the first list entry - a dict containing the PVs extra_pvs_dict = {i.split(':')[-1]: extra_pvs[i] for i in extra_pvs} return extra_pvs_dict
def read_mda(self, filename=None): if not filename: filename = self.filename source = self.mda_file_path[0].lower() if source not in ['d', 'p']: CXP.log.error("mda_file_path first character must be 'd' or 'p'") raise channel = self.mda_file_path[1] if not np.isnumeric(channel): CXP.log.error("mda_file_path second character must be numeric.") raise try: return readMDA.readMDA(filename)[2][source].data except: CXP.log.error('Could not extract array from mda file') raise
def extract_angles(fn_string, fn_range): """ Convenience function for extracting the angles from a bunch of mda files Inputs ------ fn_string: the path to the mda files with {:04d} (or similar) for specifying the number. fn_range: a two element tuple specifying the dtart and end numbers Outputs ------- A pretty list of mda number and angle which can be cut and pasted into a google doc. """ l=[] for i in range(fn_range[0], fn_range[1]): mda = readMDA(fn_string.format(i)) l.append((i, mda[0]['2xfm:userStringCalc10.DD'][2])) print 'MDA\t\tAngle (degrees)' pstr = '\t{:04d}\t\t{:3.2f}' for element in l: printpstr.format(element[0], element[1])
def getData(fname): """Extract data from mda-ASCII file and distribute into Pixel objects Returns: XAS scan data in a detector 'object' (variable "det") energy axis, transmission data array and detector filled with fluo data (detector object simply full of '0' values if no fluorescence data available) Transmission data in "trans" comprises encoder_E, I0/1/2, sample_time, encoder_angle !! I0/1/2 are already normalised to t !! """ mda = readMDA.readMDA(fname, verbose=True) scanData = mda[1] scanSize = scanData.npts try: detectorData = mda[2] detSize = detectorData.npts except: detSize = 0 det = makeDet(detSize, scanSize) # read transmission data pvColumnNames = [ 'ENERGY_RBV', 'scaler1:S2C', 'scaler1:S3C', 'scaler1:S4C', 'scaler1.T', 'BRAGG.RBV' ] trans = np.empty((len(pvColumnNames), scanSize)) for series in scanData.d: try: tag = ':'.join( series.name.split(':')[1:]) # use the PV part after the IOC id trans[pvColumnNames.index(tag)] = series.data except: pass ts = trans[pvColumnNames.index('scaler1.T')] # get the sample time "t_s" e = trans[pvColumnNames.index( 'ENERGY_RBV')] * 1000.0 # Energy axis (in eV !!) # normalise I0, I1, I2 to sample_time ts (use string "scaler1:" as identifier) for i, name in enumerate(pvColumnNames): if name.startswith('scaler1:'): trans[i] = trans[i] / ts # read fluorescence data (if it exists) if detSize != 0: print 'reading fluorescence data...' for k in range(scanSize): for i in np.arange(detSize): det[i].fpeaks[k] = detectorData.d[0].data[k][i] det[i].speaks[k] = detectorData.d[1].data[k][i] det[i].roi[k] = detectorData.d[2].data[k][i] for i in np.arange(detSize): det[i].NormT(ts) # ("ts" = sampling time per data point t_s) ## call dead time correction later; should be performed only on ## good spectra ("goodPixels") as bad spectra can contain zeros which ## this function would divide by ## call is via function: detDeadCorr(det, goodPixels) return e, trans, det
def getData(fname): """Extract data from mda-ASCII file and distribute into Pixel objects Returns: XAS scan data in a detector 'object' (variable "det") energy axis, transmission data array and detector filled with fluo data (detector object simply full of '0' values if no fluorescence data available) Transmission data in "trans" comprises encoder_E, I0/1/2, sample_time, encoder_angle !! I0/1/2 are already normalised to t !! """ # get the path to the netCDF files from the mda file mda = readMDA.readMDA(fname, verbose=False) netcdf_basename = os.path.splitext(os.path.basename(fname))[0] netcdf_directory = os.path.join(os.path.dirname(os.path.abspath(fname)), netcdf_basename) netcdf_filepattern = '{}_([0-9]*)\.nc'.format(netcdf_basename) scanData = mda[1] scanSize = scanData.npts # create and set the reader for the fluorescence detector detector_data = DetectorData(shape=(6, 6), pixelsteps_per_buffer=1, buffers_per_file=1, dirpaths=netcdf_directory, filepattern=netcdf_filepattern, mca_bins=2048, first_file_n=1) detector = Detector(detector_data) # set scanSize according to the netCDF data that was available scanSize = highest_available_scandata(detector, scanSize) detector.steprange = range(scanSize) # read transmission data # The PVs listed in pvColumnNames all refer to columnar data such as axis values. # This is contrasted by thoselisted in pvSingleValues, which are all single values. pvColumnNames = ['EncEnergy:ActPos', 'scaler1:S2C', 'scaler1:S3C', 'scaler1:S4C', 'scaler1.T', 'EncAngle:ActPos'] trans = np.empty((len(pvColumnNames), scanSize)) for series in scanData.d: try: tag = ':'.join(series.name.split(':')[1:]) # use the PV part after the IOC id if tag in pvColumnNames: trans[pvColumnNames.index(tag)] = series.data[:scanSize] except Exception as e: print e print 'missing PV ' + tag ts = trans[pvColumnNames.index('scaler1.T')] # get the sample time detector.set_ts(ts) e = trans[pvColumnNames.index('EncEnergy:ActPos')] * 1000.0 # Energy axis (in eV !!) # normalise I0, I1, I2 to sample_time ts (use string "scaler1:" as identifier) for i, name in enumerate(pvColumnNames): if name.startswith('scaler1:'): trans[i] = trans[i] / ts ## call dead time correction later; should be performed only on ## good spectra ("goodPixels") as bad spectra can contain zeros which ## this function would divide by ## call is via function: detDeadCorr(det, goodPixels) return e, trans, detector
def getData(fname): """Extract data from mda-ASCII file and distribute into Pixel objects Returns: XAS scan data in a detector 'object' (variable "det") energy axis, transmission data array and detector filled with fluo data (detector object simply full of '0' values if no fluorescence data available) Transmission data in "trans" comprises encoder_E, I0/1/2, sample_time, encoder_angle !! I0/1/2 are already normalised to t !! """ mda = readMDA.readMDA(fname, verbose=True) scanData = mda[1] scanSize = scanData.npts try: detectorData = mda[2] detSize = detectorData.npts except: detSize = 0 det = makeDet(detSize, scanSize) # generate a list of scan record detectors used (as written into the base mda file) numD = len(scanData.d) pvDetectors = [] for i in np.arange(numD): pvDetectors.append(':'.join(scanData.d[i].name.split(':')[1:])) print '' print 'Detector PVs found in source file: ', pvDetectors # test, which set of pvColumnNames is compatible with the list of scan detectors # (this is for compatibility reasons pre-/post- new DCM integration) # # before DCM install: pvColumnNames includes 'EncEnergy:ActPos' as energy PV # after DCM install: energy PV is now called 'ENERGY_RBV' # # any future changes to PV names: add a corresponding list to pvColumnNames pvColumnNames = [[ 'EncEnergy:ActPos', 'scaler1:S2C', 'scaler1:S3C', 'scaler1:S4C', 'scaler1.T', 'EncAngle:ActPos' ], [ 'ENERGY_RBV', 'scaler1:S2C', 'scaler1:S3C', 'scaler1:S4C', 'scaler1.T', 'BRAGG.RBV' ]] # go through the lists of PVs and test whether these are in the data file (i.e., in 'pvDetectors'); # choose that list which is consistent with pvDetectors pvNotFound = [] dataRead = False for PVs in pvColumnNames[:]: if dataRead: break trans = np.zeros((len(PVs), scanSize)) # # Step 1: test if list is complete for i in PVs: if i not in pvDetectors: pvNotFound.append(i) # # Step 2: if not, pass and go to next list; # if OK, go through that list again and read in transmission channels if len(pvNotFound) > 0: pvNotFound = [] pass else: print 'Reading PV columns and transmission data from source file: ', PVs print '' for i in PVs: dataIndex = pvDetectors.index(i) trans[PVs.index(i)] = scanData.d[dataIndex].data # energyPV = PVs[0] print 'Setting energy PV to: ', energyPV e = trans[PVs.index(energyPV)] * 1000.0 # Energy axis (in eV !!) # sampletimePV = 'scaler1.T' print 'Setting sample time PV to: ', sampletimePV ts = trans[PVs.index(sampletimePV)] # get the sample time "t_s" # # normalise I0, I1, I2 to sample_time ts (use string "scaler1:S" as identifier) print 'Normalising ion chamber signals (I0, I1, I2) to sample time.' for i, name in enumerate(PVs): if name.startswith('scaler1:S'): trans[i] = trans[i] / ts dataRead = True if len(pvNotFound) > 0: print 'ERROR -- one or more detector PV(s) not found: ' try: stop except: pass # for series in scanData.d: # tag = ':'.join(series.name.split(':')[1:]) # use the PV part after the Beamline/IOC id # if tag in pvColumnNames : # print 'tag found: ', tag # trans[pvColumnNames.index(tag)] = series.data # else: # print 'at least one detector PV not found (',tag,'); trying different PV list list' # read transmission data # trans = np.empty((len(pvColumnNames), scanSize)) # for series in scanData.d: # try: # tag = ':'.join(series.name.split(':')[1:]) # use the PV part after the IOC id # trans[pvColumnNames.index(tag)] = series.data # except: # pass # ts = trans[pvColumnNames.index('scaler1.T')] # get the sample time "t_s" # e = trans[pvColumnNames.index('EncEnergy:ActPos')] * 1000.0 # Energy axis (in eV !!) # read fluorescence data (if it exists) if detSize != 0: print 'reading fluorescence data...' for k in range(scanSize): for i in np.arange(detSize): det[i].fpeaks[k] = detectorData.d[0].data[k][i] det[i].speaks[k] = detectorData.d[1].data[k][i] det[i].roi[k] = detectorData.d[2].data[k][i] for i in np.arange(detSize): det[i].NormT(ts) # ("ts" = sampling time per data point t_s) ## call dead time correction later; should be performed only on ## good spectra ("goodPixels") as bad spectra can contain zeros which ## this function would divide by ## call is via function: detDeadCorr(det, goodPixels) return e, trans, det
def simple_load_test(self): fname = os.path.join(TESTDATA_DIR, MDA_FILE) mda = readMDA.readMDA(fname, verbose=False) self.assertEqual(mda[0]['rank'], 1)
def getData(fname): """Extract data from mda-ASCII file and distribute into Pixel objects Returns: XAS scan data in a detector 'object' (variable "det") energy axis, transmission data array and detector filled with fluo data (detector object simply full of '0' values if no fluorescence data available) Transmission data in "trans" comprises encoder_E, I0/1/2, sample_time, encoder_angle !! I0/1/2 are already normalised to t !! """ # get the path to the netCDF files from the mda file mda = readMDA.readMDA(fname, verbose=False) netcdf_basename = os.path.splitext(os.path.basename(fname))[0] netcdf_directory = os.path.join(os.path.dirname(os.path.abspath(fname)), netcdf_basename) netcdf_filepattern = '{}_([0-9]*)\.nc'.format(netcdf_basename) scanData = mda[1] scanSize = scanData.npts # create and set the reader for the fluorescence detector detector_data = DetectorData(shape=(6, 6), pixelsteps_per_buffer=1, buffers_per_file=1, dirpaths=netcdf_directory, filepattern=netcdf_filepattern, mca_bins=2048, first_file_n=1) detector = Detector(detector_data) # set scanSize according to the netCDF data that was available scanSize = highest_available_scandata(detector, scanSize) detector.steprange = range(scanSize) # read transmission data # The PVs listed in pvColumnNames all refer to columnar data such as axis values. # This is contrasted by thoselisted in pvSingleValues, which are all single values. pvColumnNames = [ 'EncEnergy:ActPos', 'scaler1:S2C', 'scaler1:S3C', 'scaler1:S4C', 'scaler1.T', 'EncAngle:ActPos' ] trans = np.empty((len(pvColumnNames), scanSize)) for series in scanData.d: try: tag = ':'.join( series.name.split(':')[1:]) # use the PV part after the IOC id if tag in pvColumnNames: trans[pvColumnNames.index(tag)] = series.data[:scanSize] except Exception as e: print e print 'missing PV ' + tag ts = trans[pvColumnNames.index('scaler1.T')] # get the sample time detector.set_ts(ts) e = trans[pvColumnNames.index( 'EncEnergy:ActPos')] * 1000.0 # Energy axis (in eV !!) # normalise I0, I1, I2 to sample_time ts (use string "scaler1:" as identifier) for i, name in enumerate(pvColumnNames): if name.startswith('scaler1:'): trans[i] = trans[i] / ts ## call dead time correction later; should be performed only on ## good spectra ("goodPixels") as bad spectra can contain zeros which ## this function would divide by ## call is via function: detDeadCorr(det, goodPixels) return e, trans, detector