def prepareBatchFile(self, detector, exposureTime): """ Prepares a batch file to run Best bfactors only It can be used to run in condor as a post script """ inp = open( ini.Ini().getParTestFile("BEST", "best_batch_file_template"), 'r') t = Template(inp.read()) s = t.substitute(besthome=ini.Ini().getPar("BEST", "besthome"), bestbin=ini.Ini().getPar("BEST", "best_bin"), detector=detector, exposure_time=exposureTime, folder=self.runFolder) self.completePath = os.path.join( self.runFolder, ini.Ini().getPar("BEST", "best_batch_file")) outp = open(self.completePath, 'w') outp.write(s) outp.close() # give execute permissions #os.chmod(self.completePath, 0755) self.make_exe(self.completePath) self.log.logger.debug("Batch best file created: " + self.completePath) return self.completePath
def createJob(self, firstExecutable, secondExecutable): """ Fills in the template job with the job name and saves it in the oar run folder firstExecutable : normally XDS secondExecutable : followed by best return complete job path """ inp = open(ini.Ini().getParTestFile("OAR", "oar_job_file_template"), 'r') t = Template(inp.read()) s = t.substitute(firstExecutable=firstExecutable, secondExecutable=secondExecutable) completePath = os.path.join(self.oarRunFolder, ini.Ini().getPar("OAR", "oar_job_file")) outp = open(completePath, 'w') outp.write(s) outp.close() os.system('chmod +x ' + completePath) self.log.logger.debug("OAR Job File created: " + completePath) return completePath
def parseWedges(self, wedgeNumbersListToProcess, resolution): """ Parses the XDS and Best log files and builds a dictionary with statists of corresponding wedge : self.wedgeList """ recordList = [] for i in wedgeNumbersListToProcess: record = {} record['subWedgeNumber'] = i record['resolution'] = resolution wedgeFolderPath = self.wedge.getWedgeFolderPath(i) # BEST bestLogFilePath = os.path.join( wedgeFolderPath, ini.Ini().getPar("BEST", "best_log_file")) bestRecord = self.__parseBestLogFile(bestLogFilePath) record.update(bestRecord) #pp.pprint(record) self.reciprocalMat = self.__crystGen(self.currentCell) # # get crystal type # self.crystalSystemName = self.__getCrystalSystemName(self.currentCell) # record['crystalSystemName'] = self.crystalSystemName # self.log.logger.debug("Crystal System Name: " + self.crystalSystemName) # XDS xdsLogFilePath = os.path.join( wedgeFolderPath, ini.Ini().getPar("XDS", "xds_log_file")) xdsRecord = self.__parseXdsCorrectLogFile(xdsLogFilePath) record.update(xdsRecord) xdsIntensitiesFilePath = os.path.join( wedgeFolderPath, ini.Ini().getPar("XDS", "xds_intensities_file")) # Parse hkl file and calculate resolution per reflection reflectionList = self.__parseXdsXhlFile(xdsIntensitiesFilePath) xdsRecord = self.__calculateXDSIntegratedIntensity( reflectionList, resolution) record.update(xdsRecord) if len(bestRecord) > 0 and len( xdsRecord) > 0 and reflectionList is not None and len( reflectionList) > 0: recordList.append(record) self.wedgeList = recordList
def createDagWithPostScript(self, jobFileName,postScript): """ Fills in the template gag with the job name and the post script and saves it in the condor run folder """ inp = open(ini.Ini().getParTestFile("CONDOR","dag_job_file_template"), 'r') t = Template(inp.read()) s = t.substitute(condor_job_file=jobFileName,post_script_name=postScript) completePath = os.path.join(self.condorRunFolder,ini.Ini().getPar("CONDOR","dag_job_file")) outp = open(completePath, 'w') outp.write(s) outp.close() self.log.logger.debug("DAG File created: " + completePath)
def __init__(self, load_file_name): """构造函数,参数:配置文件名""" self._ini_file = load_file_name try: open(file=load_file_name, mode="a+", encoding="utf-8") except Exception as e: logging.critical("配置文件不可读写") raise e self.ini = ini.Ini(load_file_name) self.all_dict = self.ini.get_dict() try: self.db.host = self.ini.get("database", "host") self.db.user = self.ini.get("database", "user") self.db.passwd = self.ini.get("database", "passwd") self.db.db = self.ini.get("database", "db") self.db.charset = self.ini.get("database", "charset") self.db.port = int(self.ini.get("database", "port", "3306")) logging.basicConfig( format= "[%(asctime)s] - [%(filename)s] - [%(levelname)s] : %(message)s", filename=self.ini.get("logging", "log_file"), datefmt="%Y-%m-%d %H:%M:%S", level=int(self.ini.get("logging", "level")), ) logging.critical("===============Start===============") except Exception as e: logging.critical("配置文件格式不正确,无法读取正确配置项。") raise e
def setReferenceDataSet(self, wedgeFolderName): """ wedgeFolderPath : wedge.getWedgeFolderPath(ini.Ini().getPar("XDS","xds_reference_data_set_wedge_number")) """ self.referenceDataSet = os.path.join( '..', wedgeFolderName, ini.Ini().getPar("XDS", "xds_intensities_file"))
def __init__(self, processPath, ednaPath, keysFileName): self._ednaRaddoseExecutable = os.path.join( ednaPath, ini.Ini().getPar("EDNA", "edna_raddose_executable")) self.log = localLogger.LocalLogger("raddose") self._keywords = None self.results = None self.realAbsorbedDoseRate = None self.keysFileName = keysFileName self.log.logger.debug("Using raddose keywords file: " + self.keysFileName)
def launchDag(self): """ Launches the dag file: createDag must be called before """ currentFolder = os.getcwd() os.chdir(self.condorRunFolder) if os.path.isfile(ini.Ini().getPar("CONDOR","dag_job_file")) : command = ini.Ini().getPar("CONDOR","dag_submit") + ' ' + ini.Ini().getPar("CONDOR","dag_job_file") self.log.logger.debug("Launching dag: " + command + " in " + self.condorRunFolder) # execute command and get output p = sub.Popen(command,stdout=sub.PIPE,stderr=sub.PIPE,shell=True) output, errors = p.communicate() if output is not None and len(output)>0 : self.log.logger.info(output) if errors is not None and len(errors)>0 and errors.find("Renaming rescue DAGs") < 0: self.log.logger.error("Error Launching DAG: " + command) self.log.logger.error(errors) else : self.log.logger.error( ini.Ini().getPar("CONDOR","dag_job_file") + " does not exist in " + self.condorRunFolder + ". Have you created the dag file?") os.chdir(currentFolder)
def launchJob(self): """ Launches the jobfile """ currentFolder = os.getcwd() os.chdir(self.condorRunFolder) if os.path.isfile(ini.Ini().getPar("CONDOR","condor_job_file")) : command = ini.Ini().getPar("CONDOR","condor_submit") + ' ' + ini.Ini().getPar("CONDOR","condor_job_file") self.log.logger.debug("Launching job: " + command + " in " + self.condorRunFolder) # execute command and get output p = sub.Popen(command,stdout=sub.PIPE,stderr=sub.PIPE,shell=True) output, errors = p.communicate() if output is not None and len(output)>0 : self.log.logger.info(output) if errors is not None and len(errors)>0 : self.log.logger.error("Error Launching JOB: " + command) self.log.logger.error(errors) else : self.log.logger.error( ini.Ini().getPar("CONDOR","condor_job_file") + " does not exist in " + self.condorRunFolder + ". Have you created the job file?") os.chdir(currentFolder)
def launchJob(self, jobFile): """ Launches the jobfile oarsub --stdout='job.%jobid%.out' --stderr='job.%jobid%.err' --name='Run job' -l core=1,walltime=00:03:00 /bliss/users/leal/OAR/job.sh """ currentFolder = os.getcwd() os.chdir(self.oarRunFolder) jobName = os.path.split(self.oarRunFolder)[1] if os.path.isfile(jobFile): command = ini.Ini().getPar( "OAR", "oar_submit" ) + ' --stdout=job.oar.out --stderr=job.oar.err --name=inducedRadDam' + ' -l core=1,walltime=' + ini.Ini( ).getPar("OAR", "oar_walltime") + ' ' + jobFile self.log.logger.debug("Launching job: " + command + " in " + self.oarRunFolder) # execute command and get output p = sub.Popen(command, stdout=sub.PIPE, stderr=sub.PIPE, shell=True) output, errors = p.communicate() if output is not None and len(output) > 0: self.log.logger.info(output) if errors is not None and len(errors) > 0: self.log.logger.error("Error Launching JOB: " + command) self.log.logger.error(errors) else: self.log.logger.error(ini.Ini().getPar("OAR", "oar_job_file") + " does not exist in " + self.oarRunFolder + ". Have you created the job file?") os.chdir(currentFolder)
def createJob(self, jobFileName,wedgeNumber=0): """ Fills in the template job with the job name and saves it in the condor run folder jobFileName: executable to be called return complete job path """ inp = open(ini.Ini().getParTestFile("CONDOR","condor_job_file_template"), 'r') t = Template(inp.read()) executableName = os.path.split(jobFileName)[1] s = t.substitute(executable=jobFileName,folder=self.condorRunFolder, executable_name=executableName,wedge_number=wedgeNumber) completePath = os.path.join(self.condorRunFolder,ini.Ini().getPar("CONDOR","condor_job_file")) outp = open(completePath, 'w') outp.write(s) outp.close() self.log.logger.debug("Job File created: " + completePath) return completePath
def __init__(self, name=None): # Configure logging self.logConfFileName = ini.Ini().getParTestFile( "GENERAL", "log_conf_file") logging.ColorFormatter = ColorFormatter logging.config.fileConfig(self.logConfFileName) # create logger if name == None: self.logger = logging.getLogger() else: self.logger = logging.getLogger(name)
except Usage, err: print >> sys.stderr, sys.argv[0].split("/")[-1] + ": " + str(err.msg) print >> sys.stderr, "" print >> sys.stderr, "For help use the -h or the --help option." print >> sys.stderr, "" return 2 #=============================================================================== # Fun starts here!! #=============================================================================== # Mandatory initialisations! ini.Ini(configIniFileName) myLog = localLogger.LocalLogger() # if wedgeFolderPath is None: myLog.logger.info("WedgeFolderPath is mandatory!") myLog.logger.info("Type %s -h for help" % sys.argv[0]) sys.exit(0) # get wedge details from the wedgeFolderPath wedge = wedgeHandler.WedgeHandler(wedgeFolderPath) myLog.logger.debug("Processing from %d to %d " % (firstQueueItem,lastQueueItem)) # Just an history of what has been done in this folder
linewidth=0, markeredgecolor='blue', markerfacecolor='None') plt.plot(fit.continuous_x, fit.continuous_y) plt.show() # Dose 1/2 # b.calculateTheoreticalIntensityDecay(beta= 0.8E-6,gamma = 0.01E-6,initialWilsonB=5.0) # b.getTheoreticalIntensityDecayCurve(resolution=2.5,endDose=50e6) # import matplotlib.pyplot as plt # plt.plot(b.doseRange,b.iOverI0) # plt.show() # b.getTheoreticalDOneHalf() if __name__ == "__main__": # Need this to initialise! import ini configIniFileName = 'config.ini' ini.Ini(configIniFileName) #testLinear() #testLinear1Coeff() #testMulipleLinear() #testExponentialNegative() #testMulipleExponentialReturnHalved() testTheoreticalDOneHalf()
import pack import ini import sys filename = sys.argv[1] f = file(filename) p = pack.Pack(f.read()) f.close() ini = ini.Ini(p.GetFile(sys.argv[2])) print ini
print >> sys.stderr, 'Make sure that the option -u / --unset is followed by a list separated by commas' return 2 except Usage, err: print >> sys.stderr, sys.argv[0].split("/")[-1] + ": " + str(err.msg) print >> sys.stderr, "" print >> sys.stderr, "For help use the -h or the --help option." print >> sys.stderr, "" return 2 #=============================================================================== # Fun starts here!! #=============================================================================== # Mandatory initialisations! ini.Ini(configIniFileName) myLog = localLogger.LocalLogger() # if ednaFolderName is None and (wedgeFolderPath is None or wedgeFolderTemplate is None): myLog.logger.info("InducedRadDam v0.3") myLog.logger.info( "Both ednaFolderName and wedgeFolderPath are mandatory!") sys.exit(0) # get wedge details from the wedgeFolderPath wedge = wedgeHandler.WedgeHandler(wedgeFolderPath) wedge.buildEdnaFolderPath(ednaFolderName)
import pack import sys import ini filename = sys.argv[1] #f = file('%s%s' % ('/Library/WebServer/Documents/wi/pack/', filename)) f = file(filename) p = pack.Pack(f.read()) f.close() print ini.Ini(p.GetFile(sys.argv[2]))
def __parseXdsXhlFile(self, xdsIntensitiesFilePath): """ reads the XDS_ASCII.HKL and return a list of reclections of the format: Saves the list to the file : xds_reflections_out_file h k l I sigma(I) resolution !NUMBER_OF_ITEMS_IN_EACH_DATA_RECORD=12 !ITEM_H=1 !ITEM_K=2 !ITEM_L=3 !ITEM_IOBS=4 !ITEM_SIGMA(IOBS)=5 !ITEM_XD=6 !ITEM_YD=7 !ITEM_ZD=8 !ITEM_RLP=9 !ITEM_PEAK=10 !ITEM_CORR=11 !ITEM_PSI=12 """ reflectionList = [] if os.path.exists(xdsIntensitiesFilePath) is False: self.log.logger.warning('XDS HKL file does not exist: %s' % xdsIntensitiesFilePath) return reflectionList else: inp = open(xdsIntensitiesFilePath, "r") lines = inp.readlines() inp.close() spaceGroupNumber = None header = True for line in lines: if line.find("!END_OF_DATA") >= 0: self.log.logger.debug("File parsed successfully: " + xdsIntensitiesFilePath) # Writes list of reflection to a file wedgeBasePath = os.path.dirname(xdsIntensitiesFilePath) outFilePath = os.path.join( wedgeBasePath, ini.Ini().getPar("XDS", "xds_reflections_out_file")) outFile = open(outFilePath, "wb") self.log.logger.info("Dumping reflections file to: " + outFilePath) #pp.pprint(reflectionList,depth=2) pickle.dump(reflectionList, outFile) break if header is False: # Inside the list of reflection lineContents = line.split() reflection = [int(i) for i in lineContents[:3]] intensity = float(lineContents[3]) sigma = float(lineContents[4]) resolution = self.__getD( reflection ) #self.__getD(reflection,self.currentCell, spaceGroupNumber) # put all in the same list #print '%5d%5d%5d %8.2f %.2f' %(reflection[0],reflection[1],reflection[2],float(lineContents[-1]),resolution) reflection.append(intensity) reflection.append(sigma) reflection.append(resolution) reflectionList.append(reflection) if line.find("!END_OF_HEADER") >= 0: header = False # elif line.find("SPACE_GROUP_NUMBER") >= 0 : # spaceGroupNumber = int(line.split()[1]) # #print "spaceGroupNumber ->", spaceGroupNumber return reflectionList
def prepareIniFile(self): xdsFile = os.path.join(self.xdsRunFolder, 'XDS.INP') # wait maxCycles x time.sleep for the file to be in the folder maxCycles = 120 for i in range(0, maxCycles): try: inp = open(xdsFile, "r") break except IOError: self.log.logger.info( "Waiting for the XDS.INP file to be ready... %d of %d" % (i, maxCycles)) time.sleep(1) if maxCycles == i + 1: self.log.logger.error( "Giving up... Can't open the file: " + xdsFile) sys.exit(3) lines = inp.readlines() inp.close() # create XDS.INP backup xdsBackupFileName = 'XDS.INP_' + str(time.time()) backupFile = os.path.join(self.xdsRunFolder, xdsBackupFileName) outp = open(backupFile, 'w') outp.writelines(lines) outp.close() self.log.logger.debug( "Preparing XDS.INP file. Original file backed up to: " + xdsBackupFileName) outlines = [] # Patterns to find dataRangePattern = re.compile('DATA_RANGE\s*=\s*(\d+)\s+(\d+)$') # Patterns to substitute spaceGroupNumberPattern = re.compile('SPACE_GROUP_NUMBER.*$') unitCellConstantsPattern = re.compile('UNIT_CELL_CONSTANTS.*$') backgroundRangePattern = re.compile( 'BACKGROUND_RANGE\s*=\s*(\d+)\s+(\d+)$') spotRangePattern = re.compile('SPOT_RANGE\s*=\s*(\d+)\s+(\d+)$') jobPattern = re.compile('JOB\s*=.*$') # modify file content for line in lines: line = line.strip() if line.startswith('!'): outlines.append(line) else: dataRange = dataRangePattern.search(line) spaceGroupNumber = spaceGroupNumberPattern.search(line) unitCellConstants = unitCellConstantsPattern.search(line) backgroundRange = backgroundRangePattern.search(line) spotRange = spotRangePattern.search(line) job = jobPattern.search(line) if dataRange is not None: self.firstImage = int(dataRange.group(1)) self.lastImage = int(dataRange.group(2)) self.log.logger.debug("First image = %d, Last image = %d" % (self.firstImage, self.lastImage)) outlines.append(line) elif spaceGroupNumber is not None and self.spaceGroupNumber is not None: outlines.append(' SPACE_GROUP_NUMBER= ' + str(self.spaceGroupNumber)) elif unitCellConstants is not None and self.cell is not None: outlines.append(' UNIT_CELL_CONSTANTS= ' + self.cell) elif backgroundRange is not None and self.firstImage is not None and self.lastImage is not None: if int(backgroundRange.group(2)) > self.lastImage: outlines.append(' BACKGROUND_RANGE= %d %d \n' % (self.firstImage, self.lastImage)) else: outlines.append(line) elif spotRange is not None and self.firstImage is not None and self.lastImage is not None: if int(spotRange.group(2)) > self.lastImage: outlines.append(' SPOT_RANGE= %d %d \n' % (self.firstImage, self.lastImage)) else: outlines.append(line) elif job is not None: outlines.append( ' JOB= ' + ini.Ini().getPar("XDS", "xds_job_keywords") + '\n') elif line.startswith( 'REFERENCE_DATA_SET' ): # and self.referenceDataSet is not None: # cleans previous REFERENCE_DATA_SET pass elif line.find('MINUTE=1') >= 0: # older INP XDS files # doesn't write line to the output pass else: outlines.append(line) if self.referenceDataSet is not None and self.referenceDataSet.strip( ) != '': outlines.append('REFERENCE_DATA_SET= ' + self.referenceDataSet) outp = open(xdsFile, 'w') #outp.writelines(outlines) for l in outlines: outp.write(l + '\n') outp.close()
"OAR", "oar_submit" ) + ' --stdout=job.oar.out --stderr=job.oar.err --name=inducedRadDam' + ' -l core=1,walltime=' + ini.Ini( ).getPar("OAR", "oar_walltime") + ' ' + jobFile self.log.logger.debug("Launching job: " + command + " in " + self.oarRunFolder) # execute command and get output p = sub.Popen(command, stdout=sub.PIPE, stderr=sub.PIPE, shell=True) output, errors = p.communicate() if output is not None and len(output) > 0: self.log.logger.info(output) if errors is not None and len(errors) > 0: self.log.logger.error("Error Launching JOB: " + command) self.log.logger.error(errors) else: self.log.logger.error(ini.Ini().getPar("OAR", "oar_job_file") + " does not exist in " + self.oarRunFolder + ". Have you created the job file?") os.chdir(currentFolder) if __name__ == "__main__": oarHandler = Oar('/tmp_14_days/ric/oar') jobFilePath = oarHandler.createJob(ini.Ini().getPar("XDS", "xds_bin"), '/tmp_14_days/ric/oar/best.sh') oarHandler.launchJob(jobFilePath)