def prepareBatchFile(self, detector, exposureTime):
"""
Prepares a batch file to run Best bfactors only
It can be used to run in condor as a post script
"""
inp = open(
ini.Ini().getParTestFile("BEST", "best_batch_file_template"), 'r')
t = Template(inp.read())
s = t.substitute(besthome=ini.Ini().getPar("BEST", "besthome"),
bestbin=ini.Ini().getPar("BEST", "best_bin"),
detector=detector,
exposure_time=exposureTime,
folder=self.runFolder)
self.completePath = os.path.join(
self.runFolder,
ini.Ini().getPar("BEST", "best_batch_file"))
outp = open(self.completePath, 'w')
outp.write(s)
outp.close()
# give execute permissions
#os.chmod(self.completePath, 0755)
self.make_exe(self.completePath)
self.log.logger.debug("Batch best file created: " + self.completePath)
return self.completePath
def createJob(self, firstExecutable, secondExecutable):
"""
Fills in the template job with the job name and saves it in
the oar run folder
firstExecutable : normally XDS
secondExecutable : followed by best
return complete job path
"""
inp = open(ini.Ini().getParTestFile("OAR", "oar_job_file_template"),
'r')
t = Template(inp.read())
s = t.substitute(firstExecutable=firstExecutable,
secondExecutable=secondExecutable)
completePath = os.path.join(self.oarRunFolder,
ini.Ini().getPar("OAR", "oar_job_file"))
outp = open(completePath, 'w')
outp.write(s)
outp.close()
os.system('chmod +x ' + completePath)
self.log.logger.debug("OAR Job File created: " + completePath)
return completePath
def parseWedges(self, wedgeNumbersListToProcess, resolution):
"""
Parses the XDS and Best log files and builds a dictionary
with statists of corresponding wedge : self.wedgeList
"""
recordList = []
for i in wedgeNumbersListToProcess:
record = {}
record['subWedgeNumber'] = i
record['resolution'] = resolution
wedgeFolderPath = self.wedge.getWedgeFolderPath(i)
# BEST
bestLogFilePath = os.path.join(
wedgeFolderPath,
ini.Ini().getPar("BEST", "best_log_file"))
bestRecord = self.__parseBestLogFile(bestLogFilePath)
record.update(bestRecord)
#pp.pprint(record)
self.reciprocalMat = self.__crystGen(self.currentCell)
# # get crystal type
# self.crystalSystemName = self.__getCrystalSystemName(self.currentCell)
# record['crystalSystemName'] = self.crystalSystemName
# self.log.logger.debug("Crystal System Name: " + self.crystalSystemName)
# XDS
xdsLogFilePath = os.path.join(
wedgeFolderPath,
ini.Ini().getPar("XDS", "xds_log_file"))
xdsRecord = self.__parseXdsCorrectLogFile(xdsLogFilePath)
record.update(xdsRecord)
xdsIntensitiesFilePath = os.path.join(
wedgeFolderPath,
ini.Ini().getPar("XDS", "xds_intensities_file"))
# Parse hkl file and calculate resolution per reflection
reflectionList = self.__parseXdsXhlFile(xdsIntensitiesFilePath)
xdsRecord = self.__calculateXDSIntegratedIntensity(
reflectionList, resolution)
record.update(xdsRecord)
if len(bestRecord) > 0 and len(
xdsRecord) > 0 and reflectionList is not None and len(
reflectionList) > 0:
recordList.append(record)
self.wedgeList = recordList
def createDagWithPostScript(self, jobFileName,postScript):
"""
Fills in the template gag with the job name and the post script
and saves it in the condor run folder
"""
inp = open(ini.Ini().getParTestFile("CONDOR","dag_job_file_template"), 'r')
t = Template(inp.read())
s = t.substitute(condor_job_file=jobFileName,post_script_name=postScript)
completePath = os.path.join(self.condorRunFolder,ini.Ini().getPar("CONDOR","dag_job_file"))
outp = open(completePath, 'w')
outp.write(s)
outp.close()
self.log.logger.debug("DAG File created: " + completePath)
def __init__(self, load_file_name):
"""构造函数，参数：配置文件名"""
self._ini_file = load_file_name
try:
open(file=load_file_name, mode="a+", encoding="utf-8")
except Exception as e:
logging.critical("配置文件不可读写")
raise e
self.ini = ini.Ini(load_file_name)
self.all_dict = self.ini.get_dict()
try:
self.db.host = self.ini.get("database", "host")
self.db.user = self.ini.get("database", "user")
self.db.passwd = self.ini.get("database", "passwd")
self.db.db = self.ini.get("database", "db")
self.db.charset = self.ini.get("database", "charset")
self.db.port = int(self.ini.get("database", "port", "3306"))
logging.basicConfig(
format=
"[%(asctime)s] - [%(filename)s] - [%(levelname)s] : %(message)s",
filename=self.ini.get("logging", "log_file"),
datefmt="%Y-%m-%d %H:%M:%S",
level=int(self.ini.get("logging", "level")),
)
logging.critical("===============Start===============")
except Exception as e:
logging.critical("配置文件格式不正确，无法读取正确配置项。")
raise e
def setReferenceDataSet(self, wedgeFolderName):
"""
wedgeFolderPath : wedge.getWedgeFolderPath(ini.Ini().getPar("XDS","xds_reference_data_set_wedge_number"))
"""
self.referenceDataSet = os.path.join(
'..', wedgeFolderName,
ini.Ini().getPar("XDS", "xds_intensities_file"))
def __init__(self, processPath, ednaPath, keysFileName):
self._ednaRaddoseExecutable = os.path.join(
ednaPath,
ini.Ini().getPar("EDNA", "edna_raddose_executable"))
self.log = localLogger.LocalLogger("raddose")
self._keywords = None
self.results = None
self.realAbsorbedDoseRate = None
self.keysFileName = keysFileName
self.log.logger.debug("Using raddose keywords file: " +
self.keysFileName)
def launchDag(self):
"""
Launches the dag file: createDag must be called before
"""
currentFolder = os.getcwd()
os.chdir(self.condorRunFolder)
if os.path.isfile(ini.Ini().getPar("CONDOR","dag_job_file")) :
command = ini.Ini().getPar("CONDOR","dag_submit") + ' ' + ini.Ini().getPar("CONDOR","dag_job_file")
self.log.logger.debug("Launching dag: " + command + " in " + self.condorRunFolder)
# execute command and get output
p = sub.Popen(command,stdout=sub.PIPE,stderr=sub.PIPE,shell=True)
output, errors = p.communicate()
if output is not None and len(output)>0 :
self.log.logger.info(output)
if errors is not None and len(errors)>0 and errors.find("Renaming rescue DAGs") < 0:
self.log.logger.error("Error Launching DAG: " + command)
self.log.logger.error(errors)
else :
self.log.logger.error( ini.Ini().getPar("CONDOR","dag_job_file") + " does not exist in " + self.condorRunFolder + ". Have you created the dag file?")
os.chdir(currentFolder)
def launchJob(self):
"""
Launches the jobfile
"""
currentFolder = os.getcwd()
os.chdir(self.condorRunFolder)
if os.path.isfile(ini.Ini().getPar("CONDOR","condor_job_file")) :
command = ini.Ini().getPar("CONDOR","condor_submit") + ' ' + ini.Ini().getPar("CONDOR","condor_job_file")
self.log.logger.debug("Launching job: " + command + " in " + self.condorRunFolder)
# execute command and get output
p = sub.Popen(command,stdout=sub.PIPE,stderr=sub.PIPE,shell=True)
output, errors = p.communicate()
if output is not None and len(output)>0 :
self.log.logger.info(output)
if errors is not None and len(errors)>0 :
self.log.logger.error("Error Launching JOB: " + command)
self.log.logger.error(errors)
else :
self.log.logger.error( ini.Ini().getPar("CONDOR","condor_job_file") + " does not exist in " + self.condorRunFolder + ". Have you created the job file?")
os.chdir(currentFolder)
def launchJob(self, jobFile):
"""
Launches the jobfile
oarsub --stdout='job.%jobid%.out' --stderr='job.%jobid%.err' --name='Run job' -l core=1,walltime=00:03:00 /bliss/users/leal/OAR/job.sh
"""
currentFolder = os.getcwd()
os.chdir(self.oarRunFolder)
jobName = os.path.split(self.oarRunFolder)[1]
if os.path.isfile(jobFile):
command = ini.Ini().getPar(
"OAR", "oar_submit"
) + ' --stdout=job.oar.out --stderr=job.oar.err --name=inducedRadDam' + ' -l core=1,walltime=' + ini.Ini(
).getPar("OAR", "oar_walltime") + ' ' + jobFile
self.log.logger.debug("Launching job: " + command + " in " +
self.oarRunFolder)
# execute command and get output
p = sub.Popen(command,
stdout=sub.PIPE,
stderr=sub.PIPE,
shell=True)
output, errors = p.communicate()
if output is not None and len(output) > 0:
self.log.logger.info(output)
if errors is not None and len(errors) > 0:
self.log.logger.error("Error Launching JOB: " + command)
self.log.logger.error(errors)
else:
self.log.logger.error(ini.Ini().getPar("OAR", "oar_job_file") +
" does not exist in " + self.oarRunFolder +
". Have you created the job file?")
os.chdir(currentFolder)
def createJob(self, jobFileName,wedgeNumber=0):
"""
Fills in the template job with the job name and saves it in
the condor run folder
jobFileName: executable to be called
return complete job path
"""
inp = open(ini.Ini().getParTestFile("CONDOR","condor_job_file_template"), 'r')
t = Template(inp.read())
executableName = os.path.split(jobFileName)[1]
s = t.substitute(executable=jobFileName,folder=self.condorRunFolder, executable_name=executableName,wedge_number=wedgeNumber)
completePath = os.path.join(self.condorRunFolder,ini.Ini().getPar("CONDOR","condor_job_file"))
outp = open(completePath, 'w')
outp.write(s)
outp.close()
self.log.logger.debug("Job File created: " + completePath)
return completePath
def __init__(self, name=None):
# Configure logging
self.logConfFileName = ini.Ini().getParTestFile(
"GENERAL", "log_conf_file")
logging.ColorFormatter = ColorFormatter
logging.config.fileConfig(self.logConfFileName)
# create logger
if name == None:
self.logger = logging.getLogger()
else:
self.logger = logging.getLogger(name)
except Usage, err:
print >> sys.stderr, sys.argv[0].split("/")[-1] + ": " + str(err.msg)
print >> sys.stderr, ""
print >> sys.stderr, "For help use the -h or the --help option."
print >> sys.stderr, ""
return 2
#===============================================================================
# Fun starts here!!
#===============================================================================
# Mandatory initialisations!
ini.Ini(configIniFileName)
myLog = localLogger.LocalLogger()
#
if wedgeFolderPath is None:
myLog.logger.info("WedgeFolderPath is mandatory!")
myLog.logger.info("Type %s -h for help" % sys.argv[0])
sys.exit(0)
# get wedge details from the wedgeFolderPath
wedge = wedgeHandler.WedgeHandler(wedgeFolderPath)
myLog.logger.debug("Processing from %d to %d " % (firstQueueItem,lastQueueItem))
# Just an history of what has been done in this folder
linewidth=0,
markeredgecolor='blue',
markerfacecolor='None')
plt.plot(fit.continuous_x, fit.continuous_y)
plt.show()
# Dose 1/2
# b.calculateTheoreticalIntensityDecay(beta= 0.8E-6,gamma = 0.01E-6,initialWilsonB=5.0)
# b.getTheoreticalIntensityDecayCurve(resolution=2.5,endDose=50e6)
# import matplotlib.pyplot as plt
# plt.plot(b.doseRange,b.iOverI0)
# plt.show()
# b.getTheoreticalDOneHalf()
if __name__ == "__main__":
# Need this to initialise!
import ini
configIniFileName = 'config.ini'
ini.Ini(configIniFileName)
#testLinear()
#testLinear1Coeff()
#testMulipleLinear()
#testExponentialNegative()
#testMulipleExponentialReturnHalved()
testTheoreticalDOneHalf()
import pack import ini import sys filename = sys.argv[1] f = file(filename) p = pack.Pack(f.read()) f.close() ini = ini.Ini(p.GetFile(sys.argv[2])) print ini
print >> sys.stderr, 'Make sure that the option -u / --unset is followed by a list separated by commas'
return 2
except Usage, err:
print >> sys.stderr, sys.argv[0].split("/")[-1] + ": " + str(err.msg)
print >> sys.stderr, ""
print >> sys.stderr, "For help use the -h or the --help option."
print >> sys.stderr, ""
return 2
#===============================================================================
# Fun starts here!!
#===============================================================================
# Mandatory initialisations!
ini.Ini(configIniFileName)
myLog = localLogger.LocalLogger()
#
if ednaFolderName is None and (wedgeFolderPath is None
or wedgeFolderTemplate is None):
myLog.logger.info("InducedRadDam v0.3")
myLog.logger.info(
"Both ednaFolderName and wedgeFolderPath are mandatory!")
sys.exit(0)
# get wedge details from the wedgeFolderPath
wedge = wedgeHandler.WedgeHandler(wedgeFolderPath)
wedge.buildEdnaFolderPath(ednaFolderName)
import pack
import sys
import ini
filename = sys.argv[1]
#f = file('%s%s' % ('/Library/WebServer/Documents/wi/pack/', filename))
f = file(filename)
p = pack.Pack(f.read())
f.close()
print ini.Ini(p.GetFile(sys.argv[2]))
def __parseXdsXhlFile(self, xdsIntensitiesFilePath):
"""
reads the XDS_ASCII.HKL and return a list of reclections of the format:
Saves the list to the file : xds_reflections_out_file
h k l I sigma(I) resolution
!NUMBER_OF_ITEMS_IN_EACH_DATA_RECORD=12
!ITEM_H=1
!ITEM_K=2
!ITEM_L=3
!ITEM_IOBS=4
!ITEM_SIGMA(IOBS)=5
!ITEM_XD=6
!ITEM_YD=7
!ITEM_ZD=8
!ITEM_RLP=9
!ITEM_PEAK=10
!ITEM_CORR=11
!ITEM_PSI=12
"""
reflectionList = []
if os.path.exists(xdsIntensitiesFilePath) is False:
self.log.logger.warning('XDS HKL file does not exist: %s' %
xdsIntensitiesFilePath)
return reflectionList
else:
inp = open(xdsIntensitiesFilePath, "r")
lines = inp.readlines()
inp.close()
spaceGroupNumber = None
header = True
for line in lines:
if line.find("!END_OF_DATA") >= 0:
self.log.logger.debug("File parsed successfully: " +
xdsIntensitiesFilePath)
# Writes list of reflection to a file
wedgeBasePath = os.path.dirname(xdsIntensitiesFilePath)
outFilePath = os.path.join(
wedgeBasePath,
ini.Ini().getPar("XDS", "xds_reflections_out_file"))
outFile = open(outFilePath, "wb")
self.log.logger.info("Dumping reflections file to: " +
outFilePath)
#pp.pprint(reflectionList,depth=2)
pickle.dump(reflectionList, outFile)
break
if header is False:
# Inside the list of reflection
lineContents = line.split()
reflection = [int(i) for i in lineContents[:3]]
intensity = float(lineContents[3])
sigma = float(lineContents[4])
resolution = self.__getD(
reflection
) #self.__getD(reflection,self.currentCell, spaceGroupNumber)
# put all in the same list
#print '%5d%5d%5d %8.2f %.2f' %(reflection[0],reflection[1],reflection[2],float(lineContents[-1]),resolution)
reflection.append(intensity)
reflection.append(sigma)
reflection.append(resolution)
reflectionList.append(reflection)
if line.find("!END_OF_HEADER") >= 0:
header = False
# elif line.find("SPACE_GROUP_NUMBER") >= 0 :
# spaceGroupNumber = int(line.split()[1])
# #print "spaceGroupNumber ->", spaceGroupNumber
return reflectionList
def prepareIniFile(self):
xdsFile = os.path.join(self.xdsRunFolder, 'XDS.INP')
# wait maxCycles x time.sleep for the file to be in the folder
maxCycles = 120
for i in range(0, maxCycles):
try:
inp = open(xdsFile, "r")
break
except IOError:
self.log.logger.info(
"Waiting for the XDS.INP file to be ready... %d of %d" %
(i, maxCycles))
time.sleep(1)
if maxCycles == i + 1:
self.log.logger.error(
"Giving up... Can't open the file: " + xdsFile)
sys.exit(3)
lines = inp.readlines()
inp.close()
# create XDS.INP backup
xdsBackupFileName = 'XDS.INP_' + str(time.time())
backupFile = os.path.join(self.xdsRunFolder, xdsBackupFileName)
outp = open(backupFile, 'w')
outp.writelines(lines)
outp.close()
self.log.logger.debug(
"Preparing XDS.INP file. Original file backed up to: " +
xdsBackupFileName)
outlines = []
# Patterns to find
dataRangePattern = re.compile('DATA_RANGE\s*=\s*(\d+)\s+(\d+)$')
# Patterns to substitute
spaceGroupNumberPattern = re.compile('SPACE_GROUP_NUMBER.*$')
unitCellConstantsPattern = re.compile('UNIT_CELL_CONSTANTS.*$')
backgroundRangePattern = re.compile(
'BACKGROUND_RANGE\s*=\s*(\d+)\s+(\d+)$')
spotRangePattern = re.compile('SPOT_RANGE\s*=\s*(\d+)\s+(\d+)$')
jobPattern = re.compile('JOB\s*=.*$')
# modify file content
for line in lines:
line = line.strip()
if line.startswith('!'):
outlines.append(line)
else:
dataRange = dataRangePattern.search(line)
spaceGroupNumber = spaceGroupNumberPattern.search(line)
unitCellConstants = unitCellConstantsPattern.search(line)
backgroundRange = backgroundRangePattern.search(line)
spotRange = spotRangePattern.search(line)
job = jobPattern.search(line)
if dataRange is not None:
self.firstImage = int(dataRange.group(1))
self.lastImage = int(dataRange.group(2))
self.log.logger.debug("First image = %d, Last image = %d" %
(self.firstImage, self.lastImage))
outlines.append(line)
elif spaceGroupNumber is not None and self.spaceGroupNumber is not None:
outlines.append(' SPACE_GROUP_NUMBER= ' +
str(self.spaceGroupNumber))
elif unitCellConstants is not None and self.cell is not None:
outlines.append(' UNIT_CELL_CONSTANTS= ' + self.cell)
elif backgroundRange is not None and self.firstImage is not None and self.lastImage is not None:
if int(backgroundRange.group(2)) > self.lastImage:
outlines.append(' BACKGROUND_RANGE= %d %d \n' %
(self.firstImage, self.lastImage))
else:
outlines.append(line)
elif spotRange is not None and self.firstImage is not None and self.lastImage is not None:
if int(spotRange.group(2)) > self.lastImage:
outlines.append(' SPOT_RANGE= %d %d \n' %
(self.firstImage, self.lastImage))
else:
outlines.append(line)
elif job is not None:
outlines.append(
' JOB= ' +
ini.Ini().getPar("XDS", "xds_job_keywords") + '\n')
elif line.startswith(
'REFERENCE_DATA_SET'
): # and self.referenceDataSet is not None:
# cleans previous REFERENCE_DATA_SET
pass
elif line.find('MINUTE=1') >= 0:
# older INP XDS files
# doesn't write line to the output
pass
else:
outlines.append(line)
if self.referenceDataSet is not None and self.referenceDataSet.strip(
) != '':
outlines.append('REFERENCE_DATA_SET= ' + self.referenceDataSet)
outp = open(xdsFile, 'w')
#outp.writelines(outlines)
for l in outlines:
outp.write(l + '\n')
outp.close()
"OAR", "oar_submit"
) + ' --stdout=job.oar.out --stderr=job.oar.err --name=inducedRadDam' + ' -l core=1,walltime=' + ini.Ini(
).getPar("OAR", "oar_walltime") + ' ' + jobFile
self.log.logger.debug("Launching job: " + command + " in " +
self.oarRunFolder)
# execute command and get output
p = sub.Popen(command,
stdout=sub.PIPE,
stderr=sub.PIPE,
shell=True)
output, errors = p.communicate()
if output is not None and len(output) > 0:
self.log.logger.info(output)
if errors is not None and len(errors) > 0:
self.log.logger.error("Error Launching JOB: " + command)
self.log.logger.error(errors)
else:
self.log.logger.error(ini.Ini().getPar("OAR", "oar_job_file") +
" does not exist in " + self.oarRunFolder +
". Have you created the job file?")
os.chdir(currentFolder)
if __name__ == "__main__":
oarHandler = Oar('/tmp_14_days/ric/oar')
jobFilePath = oarHandler.createJob(ini.Ini().getPar("XDS", "xds_bin"),
'/tmp_14_days/ric/oar/best.sh')
oarHandler.launchJob(jobFilePath)
