def test_deploy_calib_array():
print 80 * '_', '\nTest deploy_calib_array'
cdir = './calib'
if not os.path.exists(cdir): gu.create_directory(cdir, verb=True)
#cdir = '/reg/d/psdm/CXI/cxi83714/calib'
src = 'CxiDs1.0:Cspad.0'
type = 'pedestals'
run_start = 9991
run_end = None
arr = gu.np.ones((32, 185, 388))
cmts = {
'exp': 'cxi83714',
'ifname': 'input-file-name',
'app': 'my-app-name',
'comment': 'my-comment'
}
deploy_calib_array(cdir,
src,
type,
run_start,
run_end,
arr,
cmts,
fmt='%.1f',
pbits=3)
def makeCalibFileName(self, src, type, run_start, run_end=None):
"""Returns calibration file name.
"""
if os.path.basename(self.cdir.rstrip('/')) != 'calib':
if self.pbits & 1:
print 'WARNING! NOT calib DIRECTORY: %s' % self.cdir
return None
# there have been problems with calib-dir mounts on the mon nodes.
# raise an exception here to try to detect this problem
#assert os.path.isdir(self.cdir), 'psana calib-dir must exist: '+self.cdir
if not os.path.isdir(self.cdir):
print 'WARNING! psana calib-dir is not found: %s' % self.cdir
return None
if not self._setGroup(src):
return None
if run_start < 0:
if self.pbits & 1:
print 'WARNING! START RUN NUMBER IS NEGATIVE: %d' % run_start
return None
if run_start > 9999:
if self.pbits & 1:
print 'WARNING! START RUN NUMBER EXCEEDS 4-DIGITS: %d' % run_start
return None
if run_end is None:
self.cfname = '%d-end.data' % (run_start)
else:
if run_end < 0:
if self.pbits & 1:
print 'WARNING! END RUN NUMBER IS NEGATIVE: %d' % run_end
return None
if run_end > 9999:
if self.pbits & 1:
print 'WARNING! END RUN NUMBER IS TOO BIG: %d' % run_end
return None
if run_end < run_start:
if self.pbits & 1:
print 'WARNING! END RUN:%d < START RUN:%d' % (run_end,
run_start)
return None
self.cfname = '%d-%d.data' % (run_start, run_end)
dir = self.cdir
for subdir in (self.group, src, type):
dir = os.path.join(dir, subdir)
gu.create_directory(dir, self.pbits)
return os.path.join(dir, self.cfname)
def deploy_calib_array(cdir,
src,
type,
run_start,
run_end=None,
arr=None,
dcmts={},
fmt='%.1f',
pbits=1):
"""Deploys array in calibration file
- makes the new file name using make_calib_file_name(...)
- if file with this name already exists - rename it with current timestamp in the name
- save array in file
- add history record
"""
fname = make_calib_file_name(cdir, src, type, run_start, run_end, pbits)
path_history = '%s/HISTORY' % os.path.dirname(fname)
if os.path.exists(fname):
fname_bkp = '%s-%s' % (fname, gu.str_tstamp(fmt='%Y-%m-%dT%H:%M:%S'))
os.system('cp %s %s' % (fname, fname_bkp))
if pbits & 1:
print 'Existing file %s\nis backed-up %s' % (fname, fname_bkp)
# extend dictionary for other parameters
d = dict(dcmts)
d['run'] = run_start
d['fname'] = os.path.basename(fname)
d['src'] = src
d['ctype'] = type
# make list of comments
cmts = ['%s %s' % (k.upper().ljust(11), v) for k, v in d.iteritems()]
# save n-dimensional numpy array in the tmp text file
fntmp = tempfile.NamedTemporaryFile(mode='r+b', suffix='.data')
if pbits & 2: print 'Save constants in tmp file: %s' % fntmp.name
save_txt(fntmp.name, arr, cmts, fmt='%.1f')
if pbits & 1: print 'Deploy constants in file: %s' % fname
# USE cat in stead of cp and move in order to create output file with correct ACL permissions
cmd_cat = 'cat %s > %s' % (fntmp.name, fname)
#os.system(cmd_cat)
stream = os.popen(cmd_cat)
resp = stream.read()
msg = 'Command: %s\n - resp: %s' % (cmd_cat, resp)
if pbits & 2: print msg
# add record to the HISTORY file
hrec = _history_record(d)
if pbits & 1: print 'Add record: %sto the file: %s' % (hrec, path_history)
gu.save_textfile(hrec, path_history, mode='a')
def _common_mode_pars(self, arr_ave, arr_rms, arr_msk):
"""Returns detector-dependent common mode parameters as np.array for a few detectors and None for others.
"""
import PSCalib.GlobalUtils as gu
import math
dettype = gu.det_type_from_source(self.detname)
ave = arr_ave[arr_msk > 0].mean()
#rms = arr_ave[arr_msk>0].std()
rms = arr_rms[arr_msk > 0].mean()
logger.info('Evaluate common mode for source: %s det: %s, estimated intensity ave: %.3f rms: %.3f' %\
(self.detname, gu.dic_det_type_to_name[dettype], ave, rms))
if dettype == gu.PNCCD:
return np.array((3, math.ceil(4 * rms), math.ceil(4 * rms), 128))
#elif dettype == gu.EPIX100A:
# return np.array((4, 6, math.ceil(2*rms), math.ceil(2*rms)))
#elif dettype == gu.CSPAD:
# return np.array((1, math.ceil(3*rms), math.ceil(2*rms), 100))
#elif dettype == gu.CSPAD2X2:
# return np.array((1, math.ceil(3*rms), math.ceil(2*rms), 100))
else:
return None
def deploy_calib_file(cdir,
src,
type,
run_start,
run_end=None,
ifname='',
dcmts={},
pbits=1):
"""Deploys calibration file
- makes the new file name using make_calib_file_name(...)
- if file with this name already exists - rename it with current timestamp in the name
- save array in file
- add history record
"""
fname = make_calib_file_name(cdir, src, type, run_start, run_end, pbits)
path_history = '%s/HISTORY' % os.path.dirname(fname)
if os.path.exists(fname):
fname_bkp = '%s-%s' % (fname, gu.str_tstamp(fmt='%Y-%m-%dT%H:%M:%S'))
os.system('cp %s %s' % (fname, fname_bkp))
if pbits & 1:
print 'Existing file %s\nis backed-up %s' % (fname, fname_bkp)
# extend dictionary for other parameters
d = dict(dcmts)
d['run'] = run_start
d['fname'] = os.path.basename(fname)
d['ifname'] = ifname
d['src'] = src
d['ctype'] = type
if pbits & 1: print 'Deploy constants in file: %s' % fname
# USE cat in stead of cp and move in order to create output file with correct ACL permissions
cmd_cat = 'cat %s > %s' % (ifname, fname)
#os.system(cmd_cat)
stream = os.popen(cmd_cat)
resp = stream.read()
msg = 'Command: %s\n - resp: %s' % (cmd_cat, resp)
if pbits & 2: print msg
# add record to the HISTORY file
hrec = _history_record(d)
if pbits & 1: print 'Add record: %sto the file: %s' % (hrec, path_history)
gu.save_textfile(hrec, path_history, mode='a')
def test_deploy_calib_file():
print 80 * '_', '\nTest deploy_calib_file'
cdir = './calib'
if not os.path.exists(cdir): gu.create_directory(cdir, verb=True)
#cdir = '/reg/d/psdm/CXI/cxi83714/calib'
src = 'CxiDs1.0:Cspad.0'
type = 'geometry'
run_start = 9992
run_end = None
fname = '/reg/g/psdm/detector/alignment/cspad/calib-cxi-camera1-2014-09-24/2016-06-15-geometry-cxil0216-r150-camera1-z95mm.txt'
cmts = {'exp': 'cxi83714', 'app': 'my-app-name', 'comment': 'my-comment'}
deploy_calib_file(cdir,
src,
type,
run_start,
run_end,
fname,
cmts,
pbits=3)
def _setGroup(self, src):
"""If not available, sets group from source.
"""
if self.group == '' or self.group is None:
dettype = gu.det_type_from_source(src)
self.group = gu.dic_det_type_to_calib_group.get(dettype)
if self.group is None:
if self.pbits & 1:
print 'WARNING! CALIBRATION GROUP IS NOT FOUND FOR SOURCE %s' % src
return False
return True
def Create(self, calibdir, group, source, runnum, pbits=0, fnexpc=None, fnrepo=None, tsec=None) :
""" Factory method
Parameters
- calibdir : string - calibration directory, ex: /reg/d/psdm/AMO/amoa1214/calib
- group : string - group, ex: PNCCD::CalibV1
- source : string - data source, ex: Camp.0:pnCCD.0
- runnum : int - run number, ex: 10
- pbits=0 : int - print control bits, ex: 255
Returns
- GenericCalibPars object
"""
dettype = gu.det_type_from_source(source)
grp = group if group is not None else gu.dic_det_type_to_calib_group[dettype]
if pbits : print '%s: Detector type = %d: %s' % (self.name, dettype, gu.dic_det_type_to_name[dettype])
cbase = None
if dettype == gu.CSPAD : cbase = CalibParsBaseCSPadV1()
elif dettype == gu.CSPAD2X2 : cbase = CalibParsBaseCSPad2x2V1()
elif dettype == gu.PNCCD : cbase = CalibParsBasePnccdV1()
elif dettype == gu.PRINCETON : cbase = CalibParsBasePrincetonV1()
elif dettype == gu.ANDOR3D : cbase = CalibParsBaseAndor3dV1()
elif dettype == gu.ANDOR : cbase = CalibParsBaseAndorV1()
elif dettype == gu.EPIX100A : cbase = CalibParsBaseEpix100aV1()
elif dettype == gu.EPIX10KA : cbase = CalibParsBaseEpix10kaV1()
elif dettype == gu.JUNGFRAU : cbase = CalibParsBaseJungfrauV1()
elif dettype == gu.ACQIRIS : cbase = CalibParsBaseAcqirisV1()
elif dettype == gu.IMP : cbase = CalibParsBaseImpV1()
elif dettype in (gu.OPAL1000,\
gu.OPAL2000,\
gu.OPAL4000,\
gu.OPAL8000,\
gu.TM6740,\
gu.ORCAFL40,\
gu.FCCD960,\
gu.QUARTZ4A150,\
gu.RAYONIX,\
gu.FCCD,\
gu.TIMEPIX,\
gu.FLI,\
gu.ZYLA,\
gu.EPICSCAM,\
gu.PIMAX) : cbase = CalibParsBaseCameraV1()
else :
print '%s: calibration is not implemented data source "%s"' % (self.__class__.__name__, source)
#raise IOError('Calibration parameters for source: %s are not implemented in class %s' % (source, self.__class__.__name__))
return GenericCalibPars(cbase, calibdir, grp, source, runnum, pbits, fnexpc, fnrepo, tsec)
def get_fcalib(self):
if not isinstance(self.expName,str):
return None
if not isinstance(self.runNumber,int):
return None
if self.ds is None or self.det is None:
return None
self.det.do_reshape_2d_to_3d(flag=True)
calibFile = None
try:
import Detector.PyDetector
import PSCalib.GlobalUtils as gu
source = Detector.PyDetector.map_alias_to_source(self.detName, self.ds.env()) # 'DetInfo(CxiDs2.0:Cspad.0)'
calibSource = source.split('(')[-1].split(')')[0] # 'CxiDs2.0:Cspad.0'
detectorType = gu.det_type_from_source(source) # 1
calibGroup = gu.dic_det_type_to_calib_group[detectorType] # 'CsPad::CalibV1'
except:
return calibFile
if calibGroup is None or calibSource is None:
return calibFile
calibPath = "/reg/d/psdm/" + self.expName[0:3] + \
"/" + self.expName + "/calib/" + \
calibGroup + "/" + calibSource + "/geometry/"
if not os.path.isdir(calibPath):
return calibFile
# Determine which calib file to use
geometryFiles = os.listdir(calibPath)
# print "geometry: \n " + "\n".join(geometryFiles)
minDiff = -1e6
for fname in geometryFiles:
if fname.endswith('.data'):
endValid = False
startNum = int(fname.split('-')[0])
endNum = fname.split('-')[-1].split('.data')[0]
diff = startNum - self.runNumber
# Make sure it's end number is valid too
if 'end' in endNum:
endValid = True
else:
try:
if self.runNumber <= int(endNum):
endValid = True
except:
continue
if diff <= 0 and diff > minDiff and endValid is True:
minDiff = diff
calibFile = calibPath + fname
return calibFile
def test01():
# assuming /reg/d/psdm/CXI/cxid2714/calib/CsPad::CalibV1/CxiDs1.0:Cspad.0/pedestals/15-end.data
#cdir = '/reg/d/psdm/CXI/cxid2714/calib/'
#cdir = '/reg/d/psdm/CXI/cxi80410/calib/'
cdir = '/reg/d/psdm/CXI/cxi83714/calib/'
group = 'CsPad::CalibV1'
src = 'CxiDs1.0:Cspad.0'
type = 'pedestals'
rnum = 134
#rnum = 123456789
#--------------------------
print 80 * '_', '\nTest 1'
print 'Finding calib file for\n dir = %s\n grp = %s\n src = %s\n type= %s\n run = %d' % \
(cdir, group, src, type, rnum)
cff = CalibFileFinder(cdir, group, 0377)
fname = cff.findCalibFile(src, type, rnum)
#--------------------------
print 80 * '_', '\nTest 2'
print 'Test methods find_calib_file and make_calib_file_name'
fname_existing = find_calib_file(cdir, src, type, rnum, pbits=1)
print ' fname_existing : %s' % fname_existing
cdir = './calib'
run_start = 134
gu.create_directory(cdir, True)
fname_new = make_calib_file_name(cdir,
src,
type,
run_start,
run_end=None,
pbits=0)
print ' fname_new : %s' % fname_new
def findPsanaGeometry(experimentName, runNumber, detectorName):
ds = psana.DataSource("exp=" + experimentName + ":run=" + str(runNumber) +
':idx')
det = psana.Detector(detectorName)
det.do_reshape_2d_to_3d(flag=True)
run = ds.runs().next()
times = run.times()
evt = None
counter = 0
while evt is None:
evt = run.event(times[counter])
counter += 1
source = Detector.PyDetector.map_alias_to_source(
detectorName, ds.env()) # 'DetInfo(CxiDs2.0:Cspad.0)'
calibSource = source.split('(')[-1].split(')')[0] # 'CxiDs2.0:Cspad.0'
detectorType = gu.det_type_from_source(source) # 1
calibGroup = gu.dic_det_type_to_calib_group[
detectorType] # 'CsPad::CalibV1'
detectorName = gu.dic_det_type_to_name[detectorType].upper() # 'CSPAD'
calibPath = "/reg/d/psdm/" + experimentName[0:3] + \
"/" + experimentName + "/calib/" + \
calibGroup + "/" + calibSource + "/geometry/"
# Determine which calib file to use
geometryFiles = os.listdir(calibPath)
print "geometry: \n " + "\n".join(geometryFiles)
calibFile = None
minDiff = -1e6
for fname in geometryFiles:
if fname.endswith('.data'):
endValid = False
startNum = int(fname.split('-')[0])
endNum = fname.split('-')[-1].split('.data')[0]
diff = startNum - runNumber
# Make sure it's end number is valid too
if 'end' in endNum:
endValid = True
else:
try:
if runNumber <= int(endNum):
endValid = True
except:
continue
if diff <= 0 and diff > minDiff and endValid is True:
minDiff = diff
calibFile = calibPath + fname
return calibFile
def _history_record(dcmts):
"""Returns history record made of dictionary comments and system info
"""
user = gu.get_login()
host = gu.get_hostname()
tstamp = gu.str_tstamp(fmt='%Y-%m-%dT%H:%M:%S zone:%Z')
rnum = '%04d' % dcmts.get('run')
exp = '%s' % dcmts.get('exp')
ifname = '%s' % dcmts.get('ifname')
ofname = '%s' % dcmts.get('fname')
app = '%s' % dcmts.get('app')
cmt = '%s' % dcmts.get('comment')
return 'file:%s copy_of:%s exp:%s run:%s app:%s user:%s host:%s cptime:%s comment:%s\n' % \
(ofname.ljust(14),
ifname,
exp.ljust(8),
rnum.ljust(4),
app.ljust(10),
user,
host,
tstamp.ljust(29),
cmt)
def findPsanaGeometry(self):
try:
self.source = Detector.PyDetector.map_alias_to_source(
self.parent.detInfo,
self.parent.exp.ds.env()) # 'DetInfo(CxiDs2.0:Cspad.0)'
self.calibSource = self.source.split('(')[-1].split(')')[
0] # 'CxiDs2.0:Cspad.0'
self.detectorType = gu.det_type_from_source(self.source) # 1
self.calibGroup = gu.dic_det_type_to_calib_group[
self.detectorType] # 'CsPad::CalibV1'
self.detectorName = gu.dic_det_type_to_name[
self.detectorType].upper() # 'CSPAD'
if self.parent.args.localCalib:
self.calibPath = "./calib/" + self.calibGroup + "/" + self.calibSource + "/geometry"
else:
self.calibPath = self.parent.dir + '/' + self.parent.experimentName[:3] + '/' + \
self.parent.experimentName + "/calib/" + self.calibGroup + '/' + \
self.calibSource + "/geometry"
if self.parent.args.v >= 1: print "### calibPath: ", self.calibPath
# Determine which calib file to use
geometryFiles = os.listdir(self.calibPath)
if self.parent.args.v >= 1: print "geom: ", geometryFiles
self.calibFile = None
minDiff = -1e6
for fname in geometryFiles:
if fname.endswith('.data'):
endValid = False
startNum = int(fname.split('-')[0])
endNum = fname.split('-')[-1].split('.data')[0]
diff = startNum - self.parent.runNumber
# Make sure it's end number is valid too
if 'end' in endNum:
endValid = True
else:
try:
if self.parent.runNumber <= int(endNum):
endValid = True
except:
continue
if diff <= 0 and diff > minDiff and endValid is True:
minDiff = diff
self.calibFile = fname
except:
if self.parent.args.v >= 1: print "Couldn't find psana geometry"
self.calibFile = None
def findPsanaGeometry(self):
try:
self.source = Detector.PyDetector.map_alias_to_source(self.parent.detInfo,
self.parent.exp.ds.env()) # 'DetInfo(CxiDs2.0:Cspad.0)'
self.calibSource = self.source.split('(')[-1].split(')')[0] # 'CxiDs2.0:Cspad.0'
self.detectorType = gu.det_type_from_source(self.source) # 1
self.calibGroup = gu.dic_det_type_to_calib_group[self.detectorType] # 'CsPad::CalibV1'
self.detectorName = gu.dic_det_type_to_name[self.detectorType].upper() # 'CSPAD'
if self.parent.args.localCalib:
self.calibPath = "./calib/" + self.calibGroup + "/" + self.calibSource + "/geometry"
else:
self.calibPath = "/reg/d/psdm/" + self.parent.experimentName[0:3] + \
"/" + self.parent.experimentName + "/calib/" + \
self.calibGroup + "/" + self.calibSource + "/geometry"
if self.parent.args.v >= 1: print "### calibPath: ", self.calibPath
# Determine which calib file to use
geometryFiles = os.listdir(self.calibPath)
if self.parent.args.v >= 1: print "geom: ", geometryFiles
self.calibFile = None
minDiff = -1e6
for fname in geometryFiles:
if fname.endswith('.data'):
endValid = False
startNum = int(fname.split('-')[0])
endNum = fname.split('-')[-1].split('.data')[0]
diff = startNum - self.parent.runNumber
# Make sure it's end number is valid too
if 'end' in endNum:
endValid = True
else:
try:
if self.parent.runNumber <= int(endNum):
endValid = True
except:
continue
if diff <= 0 and diff > minDiff and endValid is True:
minDiff = diff
self.calibFile = fname
except:
if self.parent.args.v >= 1: print "Couldn't find psana geometry"
self.calibFile = None