def runValidationCryoEMStep(self):
version = Plugin.getPhenixVersion()
if version == '1.13':
print("PHENIX version: 1.13")
else:
print("PHENIX version: ", version)
fileName = self.inputStructure.get().getFileName()
atomStruct = os.path.abspath(fileName)
# starting volume (.mrc)
if self.vol is not None:
tmpMapFile = self.VALIDATIONCRYOEMFILE
volume = os.path.abspath(self._getExtraPath(tmpMapFile))
else:
volume = None
# MolProbity is run to get the file molprobity.out
# (necessary to get geometry outliers)
numberOfThreads = self.numberOfThreads.get()
args = self._writeArgsMolProbity(atomStruct, vol=volume)
cwd = os.getcwd() + "/" + self._getExtraPath()
# script with auxiliary files
retry(Plugin.runPhenixProgram, Plugin.getProgram(MOLPROBITY),
args, cwd=cwd, listAtomStruct=[atomStruct], log=self._log)
args = self._writeArgsValCryoEM(atomStruct, volume, self.vol)
if Plugin.getPhenixVersion() != PHENIXVERSION and self.vol is not None:
retry(Plugin.runPhenixProgram, Plugin.getProgram(VALIDATION_CRYOEM),
args, cwd=cwd, listAtomStruct=[atomStruct], log=self._log)
def runMolprobityStep(self):
version = Plugin.getPhenixVersion()
if version == '1.13':
print("PHENIX version: 1.13")
else:
print(("PHENIX version: ", version))
# PDBx/mmCIF
fileName = self.inputStructure.get().getFileName()
# self.atomStruct = os.path.abspath(fileName)
self.atomStruct = os.getcwd() + "/" + fileName
# starting volume (.mrc)
if (self.inputVolume.get() or self.inputStructure.get().getVolume()) \
is not None:
tmpMapFile = self.MOLPROBITYFILE
# self.vol = os.path.abspath(self._getExtraPath(tmpMapFile))
self.vol = os.getcwd() + "/" + self._getExtraPath(tmpMapFile)
args = self._writeArgsMolProbityExpand(self.atomStruct, self.vol)
else:
args = self._writeArgsMolProbityExpand(self.atomStruct, vol=None)
# script with auxiliary files
retry(
Plugin.runPhenixProgram,
Plugin.getProgram(MOLPROBITY),
# args, cwd=os.path.abspath(self._getExtraPath()),
args,
cwd=self._getExtraPath(),
listAtomStruct=[self.atomStruct],
log=self._log)
def _validate(self):
errors = []
# Check that the program exists
program = Plugin.getProgram(EMRINGER)
if program is None:
errors.append("Missing variables EMRINGER and/or PHENIX_HOME")
elif not os.path.exists(program):
errors.append("Binary '%s' does not exists.\n" % program)
elif not self.is_tool(DISPLAY):
errors.append("display program missing.\n "
"Please install imagemagick package")
# If there is any error at this point it is related to config variables
if errors:
errors.append("Check configuration file: " +
Config.SCIPION_LOCAL_CONFIG)
errors.append("and set EMRINGER and PHENIX_HOME variables "
"properly.")
if program is not None:
errors.append("Current values:")
errors.append("PHENIX_HOME = %s" % Plugin.getVar(PHENIX_HOME))
errors.append("EMRINGER = %s" % EMRINGER)
# Check that the input volume exist
if self._getInputVolume() is None:
errors.append("Error: You should provide a volume.\n")
return errors
def _showRingerResults(self, e=None):
i = int(self.residue)
index = int(self.dataDict['_residues_dict'][self.residueList[i]])
# emringer main file
mainDataFile = glob.glob(
self.protocol._getExtraPath("*_emringer.pkl"))[0]
command = """from mmtbx.ringer.em_rolling import easy_pickle
import matplotlib.pyplot as plt
# process file '%s'
file_name='%s'
ringer_results = easy_pickle.load(file_name)
figure = plt.figure() # (figsize=(20,1000))
def show_residue (residue, show_background_boxes=True) :
subplots = []
for i in range(1, residue.n_chi + 1) :
chi = residue.get_angle(i)
if (chi is None) : continue
if (len(subplots) > 0) :
p = figure.add_subplot(4, 1, i, sharex=subplots[0])
else:
p = figure.add_subplot(4, 1, i)
p.set_title(residue.format())
p.set_position([0.15, 0.725 - 0.225*(i-1), 0.8, 0.225])
x = [ k*chi.sampling for k in range(len(chi.densities)) ]
p.plot(x, chi.densities, 'r-', linewidth=1)
if (chi.fofc_densities is not None) :
p.plot(x, chi.fofc_densities, linestyle='--', color=[0.5,0.0,1.0])
p.axvline(chi.angle_current, color='b', linewidth=2, linestyle='--')
p.axhline(0, color=(0.4,0.4,0.4), linestyle='--', linewidth=1)
if show_background_boxes:
p.axhspan(0.3,1,facecolor="green",alpha=0.5)
p.axhspan(-1,0.3,facecolor="grey",alpha=0.5)
p.set_xlim(0,360)
p.set_ylabel("Rho")
p.set_xlabel("Chi" + str(i))
subplots.append(p)
plt.subplots_adjust(left = 0.18, bottom = 0.00, right = 0.94,
top = 0.93,
wspace = 0.80, hspace = 0.43)
plt.tight_layout()
plt.show()
index = %d
show_residue(ringer_results[index])
""" % (mainDataFile, mainDataFile, index)
with open(self.EMRINGERSUBPLOTSFILENAME, "w") as f:
f.write(command)
# execute file with phenix.python
Plugin.runPhenixProgram("", self.EMRINGERSUBPLOTSFILENAME)
def runSuperposePDBsStep(self):
args = os.path.abspath(self.inputStructureFixed.get().getFileName())
args += " "
args += os.path.abspath(self.inputStructureMoving.get().getFileName())
cwd = os.getcwd() + "/" + self._getExtraPath()
try:
Plugin.runPhenixProgram(Plugin.getProgram(SUPERPOSE), args,
extraEnvDict=None, cwd=cwd)
except:
# This exception will run when using phenix v. 1.16 after running
# real space refine the .cif file generated can not be recognized by
# superpose pdbs program and an error is produced
list_args = args.split()
self._runChangingCifFormatSuperpose(list_args)
def testPhenixRSRefineFromPDB(self):
""" This test checks that phenix real space refine protocol runs
with an atomic structure; No Volume was provided and an error message
is expected
"""
print("Run phenix real space refine protocol from imported pdb file "
"without imported or pdb-associated volume")
# import PDB
structure_refmac3 = self._importStructRefmac3()
self.assertTrue(structure_refmac3.getFileName())
self.assertFalse(structure_refmac3.getVolume())
# real_space_refine
args = {
'resolution': 3.5,
'inputStructure': structure_refmac3,
'numberOfThreads': 4
}
if Plugin.getPhenixVersion() == PHENIXVERSION:
args['doSecondary'] = False
protRSRefine = self.newProtocol(PhenixProtRunRSRefine, **args)
protRSRefine.setObjLabel('RSRefine without\nvolume, should NOT work')
try:
self.launchProtocol(protRSRefine)
except Exception as e:
self.assertTrue(
True, "This test should return a error message as '"
" Input volume cannot be EMPTY.'\n")
return
self.assertTrue(False)
def testgetVersion(self):
version = Plugin.getPhenixVersion()
if version == PHENIXVERSION:
self.assertEqual(version, PHENIXVERSION)
else:
print(("Your version is not " + PHENIXVERSION + " anymore"))
print(("Your current version is " + version))
def _readValidationPklFile(self, fileName):
self.SUMMARYFILENAME = self._getTmpPath(self.SUMMARYFILENAME)
command = """import pickle
import collections
import json
def pickleData(file):
with open(file,"r") as f:
return pickle.load(f)
# process file {VALIDATIONCRYOEMPKLFILENAME}"
data = pickleData('{VALIDATIONCRYOEMPKLFILENAME}')
dictSummary = collections.OrderedDict()
dictSummary['Rhama_Outliers'] = data.model.geometry.ramachandran.outliers
dictSummary['Rhama_Favored'] = data.model.geometry.ramachandran.favored
dictSummary['Rota_Outliers'] = data.model.geometry.rotamer.outliers
dictSummary['Cbeta_Outliers_n'] = data.model.geometry.c_beta.cbetadev.n_outliers
dictSummary['Clash_score'] = data.model.geometry.clash.score
dictSummary['MolProbity_score'] = data.model.geometry.molprobity_score
""".format(VALIDATIONCRYOEMPKLFILENAME=fileName)
command += """with open('%s',"w") as f:
f.write(json.dumps(dictSummary))
""" % (self.SUMMARYFILENAME)
pythonFileName = self.SUMMARYFILENAME.replace('.txt', '.py')
# write script file
with open(pythonFileName, "w") as f:
f.write(command)
# execute file with phenix.python
Plugin.runPhenixProgram("", pythonFileName)
# read file in scipion python
with open(self.SUMMARYFILENAME, "r") as f:
dictSummary = f.read()
dictSummary = json.loads(dictSummary,
object_pairs_hook=collections.OrderedDict)
self.ramachandranOutliers = Float(dictSummary['Rhama_Outliers'])
self.ramachandranFavored = Float(dictSummary['Rhama_Favored'])
self.rotamerOutliers = Float(dictSummary['Rota_Outliers'])
self.cbetaOutliers = Integer(dictSummary['Cbeta_Outliers_n'])
self.clashscore = Float(dictSummary['Clash_score'])
self.overallScore = Float(dictSummary['MolProbity_score'])
def _writeArgsMolProbityExpand(self, atomStruct, vol=None):
args = self._writeArgsMolProbity(atomStruct, vol)
if Plugin.getPhenixVersion() != PHENIXVERSION:
args += " pickle=True"
args += " pdb_interpretation.clash_guard.nonbonded_distance_threshold=None"
args += " %s " % self.extraParams.get()
# args += " wxplots=True" # TODO: Avoid the direct opening of plots
return args
def _validate(self):
errors = []
# Check that the program exists
program = Plugin.getProgram(SUPERPOSE)
if not os.path.exists(program):
errors.append("Cannot find " + program)
# If there is any error at this point it is related to config variables
errors.append("Check configuration file: " +
Config.SCIPION_LOCAL_CONFIG)
errors.append("and set PHENIX_HOME variables properly.")
if program is not None:
errors.append("Current values:")
errors.append("PHENIX_HOME = %s" % Plugin.getVar(PHENIX_HOME))
errors.append("SUPERPOSE = %s" % SUPERPOSE)
return errors
def _insertAllSteps(self):
self._insertFunctionStep('convertInputStep', self.REALSPACEFILE)
self._insertFunctionStep('runRSrefineStep', self.REALSPACEFILE)
self._insertFunctionStep('runMolprobityStep', self.REALSPACEFILE)
if Plugin.getPhenixVersion() != PHENIXVERSION:
self._insertFunctionStep('runValidationCryoEMStep',
self.REALSPACEFILE)
self._insertFunctionStep('createOutputStep')
def runRSrefineStep(self, tmpMapFile):
atomStruct = os.path.abspath(self.inputStructure.get().getFileName())
vol = os.path.abspath(self._getExtraPath(tmpMapFile))
args = self._writeArgsRSR(atomStruct, vol)
cwd = os.getcwd() + "/" + self._getExtraPath()
retry(
Plugin.runPhenixProgram,
Plugin.getProgram(REALSPACEREFINE),
args,
# cwd=os.path.abspath(self._getExtraPath()),
cwd=cwd,
listAtomStruct=[atomStruct],
log=self._log)
self.refinedFile = False
for item in os.listdir(self._getExtraPath()):
p = re.compile('\d+')
if p.search(item) is not None and item.endswith(".cif"):
self.refinedFile = True
break
if self.refinedFile == False:
print("WARNING!!!\nPHENIX error:\n pdb_interpretation.clash_guard" \
" failure: High number of nonbonded interaction distances " \
"< 0.5. This error has been disable by running the same " \
"command with the same following additional " \
"argument:\npdb_interpretation.clash_guard." \
"nonbonded_distance_threshold=None ")
args += " pdb_interpretation.clash_guard." \
"nonbonded_distance_threshold=None"
retry(
Plugin.runPhenixProgram,
Plugin.getProgram(REALSPACEREFINE),
args,
# cwd=os.path.abspath(self._getExtraPath()),
cwd=cwd,
listAtomStruct=[atomStruct],
log=self._log)
def _writeArgsMolProbity(self, atomStruct, vol=None):
args = ""
args += atomStruct
if Plugin.getPhenixVersion() == PHENIXVERSION:
args += " "
args += " map_file_name=%s" % vol
args += " pickle=True"
args += " "
args += " d_min=%f" % self.resolution.get()
args += " "
numberOfThreads = self.numberOfThreads.get()
if numberOfThreads > 1:
args += " nproc=%d" % numberOfThreads
return args
def _writeArgsRSR(self, atomStruct, vol):
if Plugin.getPhenixVersion() == PHENIXVERSION19:
args = " "
else:
args = " model_file="
args += "%s " % atomStruct
if Plugin.getPhenixVersion() == PHENIXVERSION19:
args += " "
else:
args += " map_file="
args += "%s " % vol
args += " resolution=%f" % self.resolution
args += " secondary_structure.enabled=%s" % self.doSecondary
args += " run="
if self.minimizationGlobal == True:
args += "minimization_global+"
if self.rigidBody == True:
args += "rigid_body+"
if self.localGridSearch == True:
args += "local_grid_search+"
if self.morphing == True:
args += "morphing+"
if self.simulatedAnnealing == True:
args += "simulated_annealing+"
if self.adp == True:
args += "adp+"
args = args[:-1]
# args += " run=minimization_global+local_grid_search+morphing+simulated_annealing"
args += " macro_cycles=%d" % self.macroCycles
args += " model_format=pdb+mmcif"
# args += " write_pkl_stats=True"
args += " %s " % self.extraParams.get()
numberOfThreads = self.numberOfThreads.get()
if numberOfThreads > 1:
args += " nproc=%d" % numberOfThreads
return args
def validateBase(self, program, label):
errors = []
# Check that the program exists
program = Plugin.getProgram(program)
if program is None:
errors.append("Missing variables %s and/or PHENIX_HOME" % label)
elif not os.path.exists(program):
errors.append("Binary '%s' does not exists.\n" % program)
errors.append("Check if you need to upgrade your PHENIX version"
" to run %s.\nYour current PHENIX version is %s.\n" %
(label, PHENIXVERSION))
# If there is any error at this point it is related to config variables
if errors:
errors.append("Check configuration file: " +
Config.SCIPION_LOCAL_CONFIG)
errors.append("and set %s and PHENIX_HOME variables "
"properly." % label)
if program is not None:
errors.append("Current values:")
errors.append("PHENIX_HOME = %s" % Plugin.getHome())
#errors.append("%s = %s" % label, program)
return errors
def runMolprobityStep(self, tmpMapFile):
# PDBx/mmCIF
self._getRSRefineOutput()
atomStruct = os.path.abspath(self.outAtomStructName)
# starting volume (.mrc)
vol = os.path.abspath(self._getExtraPath(tmpMapFile))
args = self._writeArgsMolProbity(atomStruct, vol)
cwd = os.getcwd() + "/" + self._getExtraPath()
retry(
Plugin.runPhenixProgram,
Plugin.getProgram(MOLPROBITY2),
# args, cwd=os.path.abspath(self._getExtraPath()),
args,
cwd=cwd,
listAtomStruct=[atomStruct],
log=self._log)
def runDockInMapStep(self):
# starting structure
atomStructFileName = self.inputStructure.get().getFileName()
atomStruct = os.getcwd() + "/" + atomStructFileName
# starting map (.mrc)
mapFile = self.DOCKINMAPFILE
vol = os.getcwd() + "/" + self._getExtraPath(mapFile)
args = self._writeArgsDocKInMap(vol, atomStruct)
cwd = os.getcwd() + "/" + self._getExtraPath()
retry(
Plugin.runPhenixProgram,
Plugin.getProgram(DOCKINMAP),
# args, cwd=os.path.abspath(self._getExtraPath()),
args,
cwd=cwd,
listAtomStruct=[atomStruct],
log=self._log)
def testMolProbityValidationSeveralChains(self):
""" This test checks that MolProbity validation protocol runs with a
volume provided directly as inputVol, the input PDB was fitted to
the volume and refined previously by coot and refmac in another project
"""
print("Run MolProbity validation from imported volume and pdb file " \
"already refined by Coot and Refmac in another project")
# Import Volume
volume3 = self._importVolume3()
# import PDB
structure5_PDB = self._importStructurePDBWoVol2()
# MolProbity
args = {
'inputVolume': volume3,
'resolution': 2.2,
'inputStructure': structure5_PDB,
'numberOfThreads': 4
}
protMolProbity = self.newProtocol(PhenixProtRunMolprobity, **args)
protMolProbity.setObjLabel('MolProbity validation\n volume and pdb\n')
self.launchProtocol(protMolProbity)
# check MolProbity results
if Plugin.getPhenixVersion() == PHENIXVERSION18:
self.checkResults(ramOutliers=0.12,
ramFavored=95.86,
rotOutliers=0.52,
cbetaOutliers=0,
clashScore=9.70,
overallScore=1.80,
protMolProbity=protMolProbity)
else:
self.checkResults(ramOutliers=0.12,
ramFavored=95.86,
rotOutliers=0.52,
cbetaOutliers=0,
clashScore=9.74,
overallScore=1.80,
protMolProbity=protMolProbity)
def runValidationCryoEMStep(self, tmpMapFile):
# PDBx/mmCIF
atomStruct = os.path.abspath(self.outAtomStructName)
# starting volume (.mrc)
volume = os.path.abspath(self._getExtraPath(tmpMapFile))
if self.inputVolume.get() is not None:
vol = self.inputVolume.get()
else:
vol = self.inputStructure.get().getVolume()
args = self._writeArgsValCryoEM(atomStruct, volume, vol)
cwd = os.getcwd() + "/" + self._getExtraPath()
retry(
Plugin.runPhenixProgram,
Plugin.getProgram(VALIDATION_CRYOEM),
# args, cwd=os.path.abspath(self._getExtraPath()),
args,
cwd=cwd,
listAtomStruct=[atomStruct],
log=self._log)
def runEMRingerStep(self):
inPDBAtomStructFile = self.inputStructure.get().getFileName()
# atomStruct = os.path.abspath(inPDBAtomStructFile)
if not inPDBAtomStructFile == os.path.abspath(inPDBAtomStructFile):
atomStruct = os.getcwd() + "/" + inPDBAtomStructFile
else:
atomStruct = inPDBAtomStructFile
# vol = os.path.abspath(self._getExtraPath(self.EMRINGERFILE))
vol = os.getcwd() + "/" + self._getExtraPath(self.EMRINGERFILE)
args = self._writeArgsEMRinger(atomStruct, vol)
# script with auxiliary files
import sys
sys.stdout.flush()
# import time
# time.sleep(30)
retry(
Plugin.runPhenixProgram,
Plugin.getProgram(EMRINGER),
# args, cwd=os.path.abspath(self._getExtraPath()),
args,
cwd=self._getExtraPath(),
listAtomStruct=[atomStruct],
log=self._log)
def createOutputStep(self):
# self._getRSRefineOutput()
pdb = AtomStruct()
pdb.setFileName(self.outAtomStructName)
if self.inputVolume.get() is not None:
pdb.setVolume(self.inputVolume.get())
else:
pdb.setVolume(self.inputStructure.get().getVolume())
self._defineOutputs(outputPdb=pdb)
self._defineSourceRelation(self.inputStructure.get(), pdb)
if self.inputVolume.get() is not None:
self._defineSourceRelation(self.inputVolume.get(), pdb)
if Plugin.getPhenixVersion() == PHENIXVERSION:
MOLPROBITYOUTFILENAME = self._getExtraPath(
self.MOLPROBITYOUTFILENAME)
self._parseFile(MOLPROBITYOUTFILENAME)
else:
VALIDATIONCRYOEMPKLFILENAME = self._getExtraPath(
self.VALIDATIONCRYOEMPKLFILE)
self._readValidationPklFile(VALIDATIONCRYOEMPKLFILENAME)
self._store()
def createOutputStep(self):
MOLPROBITYOUTFILENAME = self._getExtraPath(self.MOLPROBITYOUTFILENAME)
try:
self._parseFile(MOLPROBITYOUTFILENAME)
except:
if self.MOLPROBITYFILE is not None:
# self.vol = os.path.abspath(self._getExtraPath(self.MOLPROBITYFILE))
self.vol = self._getExtraPath(self.MOLPROBITYFILE)
args = self._writeArgsMolProbityExpand(self.atomStruct,
self.vol)
else:
args = self._writeArgsMolProbityExpand(self.atomStruct,
vol=None)
args += " allow_polymer_cross_special_position=True "
retry(
Plugin.runPhenixProgram,
Plugin.getProgram(MOLPROBITY),
# args, cwd=os.path.abspath(self._getExtraPath()),
args,
cwd=self._getExtraPath(),
listAtomStruct=[self.atomStruct],
log=self._log)
self._parseFile(MOLPROBITYOUTFILENAME)
self._store()
def testValCryoEMFFromVolumeAndPDB4(self):
""" This test checks that phenix real_space_refine protocol runs
with a volume provided directly as inputVol (with half1 and half2)
and the input PDB from data banks; default refine strategy;
(MolProbity has been run to
compare values before and after refinement). MolProbity and
ValCryoEM after RSRefine should show identical values.
"""
print("Run phenix real_space_refine from imported volume and pdb file "
"from data banks (vol origin 0.0, 0.0, 0.0); default refine "
"strategy; (MolProbity has been run to compare values before "
"and after refinement). MolProbity and ValCryoEM after RSRefine "
"should show identical values.")
# Import Volume
volume_hemo_half1_hal2 = self._importHemoHalf1Half2()
# import PDB
structure_hemo_pdb = self._importStructHemoPDB()
# MolProbity
args = {
'inputVolume': volume_hemo_half1_hal2,
'resolution': 3.2,
'inputStructure': structure_hemo_pdb,
'numberOfThreads': 4
}
protMolProbity = self.newProtocol(PhenixProtRunMolprobity, **args)
protMolProbity.setObjLabel(
'MolProbity validation\n'
'full and half volumes\nand pdb\nhemoglobin')
self.launchProtocol(protMolProbity)
# check MolProbity results
self.checkMPResults(ramOutliers=0.00,
ramFavored=95.23,
rotOutliers=0.43,
cbetaOutliers=0,
clashScore=3.53,
overallScore=1.48,
protMolProbity=protMolProbity)
# real_space_refine
args = {
'inputVolume': volume_hemo_half1_hal2,
'resolution': 3.2,
'inputStructure': structure_hemo_pdb,
'numberOfThreads': 4
}
if Plugin.getPhenixVersion() == PHENIXVERSION:
args['doSecondary'] = False
protRSRefine = self.newProtocol(PhenixProtRunRSRefine, **args)
protRSRefine.setObjLabel(
'RSRefine hemo\n emd_3488.map (full, half1, half2)\nand '
'5ni1.pdb\n')
self.launchProtocol(protRSRefine)
# check real_space_refine results
if Plugin.getPhenixVersion() == PHENIXVERSION:
self.checkRSRefineResults(ramOutliers=0.00,
ramFavored=97.53,
rotOutliers=0.00,
cbetaOutliers=0,
clashScore=2.43,
overallScore=1.12,
protRSRefine=protRSRefine)
else:
self.checkRSRefineResults(ramOutliers=0.00,
ramFavored=96.11,
rotOutliers=3.04,
cbetaOutliers=0,
clashScore=5.30,
overallScore=1.92,
protRSRefine=protRSRefine)
# MolProbity2
args = {
'inputVolume': volume_hemo_half1_hal2,
'resolution': 3.2,
'inputStructure': protRSRefine.outputPdb,
'numberOfThreads': 4
}
protMolProbity2 = self.newProtocol(PhenixProtRunMolprobity, **args)
protMolProbity2.setObjLabel(
'MolProbity validation\n'
'full and half volumes\nand pdb\nhemoglobin')
self.launchProtocol(protMolProbity2)
# check MolProbity results
if Plugin.getPhenixVersion() == PHENIXVERSION:
self.checkMPResults(ramOutliers=0.00,
ramFavored=97.53,
rotOutliers=0.00,
cbetaOutliers=0,
clashScore=2.43,
overallScore=1.12,
protMolProbity=protMolProbity2)
elif Plugin.getPhenixVersion() == PHENIXVERSION18:
self.checkMPResults(ramOutliers=0.00,
ramFavored=96.64,
rotOutliers=4.77,
cbetaOutliers=0,
clashScore=6.07,
overallScore=2.06,
protMolProbity=protMolProbity2)
elif Plugin.getPhenixVersion() == PHENIXVERSION19:
self.checkMPResults(ramOutliers=0.00,
ramFavored=97.35,
rotOutliers=2.39,
cbetaOutliers=0,
clashScore=3.87,
overallScore=1.59,
protMolProbity=protMolProbity2)
else:
self.checkMPResults(ramOutliers=0.00,
ramFavored=96.11,
rotOutliers=3.04,
cbetaOutliers=0,
clashScore=5.30,
overallScore=1.92,
protMolProbity=protMolProbity2)
# validation_cryoEM
args = {
'inputVolume': volume_hemo_half1_hal2,
'resolution': 3.2,
'inputStructure': protRSRefine.outputPdb
}
protValCryoEM = self.newProtocol(PhenixProtRunValidationCryoEM, **args)
protValCryoEM.setObjLabel(
'ValCryoEM\nafter RSRefine\nvolume_hemo_org and '
'protRSRefine.outputPdb\n')
try:
self.launchProtocol(protValCryoEM)
except Exception as e:
self.assertTrue(
True, "This test should return a error message as '"
" Protocol has validation errors:\n"
"Binary '/usr/local/phenix-1.13-2998/modules/phenix/phenix/"
"command_line/validation_cryoem.py' does not exists.\n"
"Check if you need to upgrade your PHENIX version to run "
"VALIDATION_CRYOEM.\nYour current PHENIX version is 1.13.\n"
"Check configuration file: " + Config.SCIPION_LOCAL_CONFIG +
"\n"
"and set VALIDATION_CRYOEM and PHENIX_HOME variables properly.\n"
"Current values:\nPHENIX_HOME = /usr/local/phenix-1.13-2998\n")
return
# self.assertTrue(False)
# check validation cryoem results
self.checkValCryoEMResults(ramOutliers=0.00,
ramFavored=96.11,
rotOutliers=3.04,
cbetaOutliers=0,
clashScore=5.30,
overallScore=1.92,
protValCryoEM=protValCryoEM)
def testValCryoEMFFromVolumeAssociatedToPDB3(self):
""" This test checks that phenix real_space_refine protocol runs
with a volume provided directly as inputVol and the input PDB from
data banks; default refine strategy; (MolProbity has been run to
compare values before and after refinement). MolProbity and
ValCryoEM after RSRefine should show identical values.
"""
print("Run phenix real_space_refine from imported volume and pdb file "
"from data banks (vol origin 0.0, 0.0, 0.0); default refine "
"strategy; (MolProbity has been run to compare values before "
"and after refinement). MolProbity and ValCryoEM after RSRefine "
"should show identical values.")
# import PDB
structure_nucleosome_pdb = self._importStructNucleosomePDB()
# MolProbity
args = {
'inputStructure': structure_nucleosome_pdb,
'numberOfThreads': 4
}
protMolProbity = self.newProtocol(PhenixProtRunMolprobity, **args)
protMolProbity.setObjLabel('MolProbity validation\n'
'volume and pdb\nnucleosome')
self.launchProtocol(protMolProbity)
# check MolProbity results
self.checkMPResults(ramOutliers=1.77,
ramFavored=87.62,
rotOutliers=10.40,
cbetaOutliers=2,
clashScore=12.17,
overallScore=2.98,
protMolProbity=protMolProbity)
# real_space_refine
args = {
'resolution': 4.0,
'inputStructure': structure_nucleosome_pdb,
'numberOfThreads': 4
}
if Plugin.getPhenixVersion() == PHENIXVERSION:
args['doSecondary'] = False
protRSRefine = self.newProtocol(PhenixProtRunRSRefine, **args)
protRSRefine.setObjLabel('RSRefine nucleosome\nvolume and ' 'pdb\n')
# TODO, this protocol fails because
# Opening quote in middle of word: ATOM 5963 O5' . DA I -72 ? 73.27900 73.22500 141.39500 1.000 212.95366 O ? L ? . 1
# so 05' is not valid for the cif reader, I think is fair and should
# not be counted as a problem
self.launchProtocol(
protRSRefine) #Keep this line commented in pull requests
# return; self.launchProtocol(protRSRefine)
# check real_space_refine results
if Plugin.getPhenixVersion() == PHENIXVERSION:
self.checkRSRefineResults(ramOutliers=0.00,
ramFavored=95.92,
rotOutliers=0.16,
cbetaOutliers=0,
clashScore=7.46,
overallScore=1.69,
protRSRefine=protRSRefine)
elif Plugin.getPhenixVersion() == PHENIXVERSION18:
self.checkRSRefineResults(ramOutliers=0.00,
ramFavored=96.19,
rotOutliers=12.32,
cbetaOutliers=0,
clashScore=12.40,
overallScore=2.69,
protRSRefine=protRSRefine)
elif Plugin.getPhenixVersion() == PHENIXVERSION19:
self.checkRSRefineResults(ramOutliers=0.00,
ramFavored=98.23,
rotOutliers=2.72,
cbetaOutliers=0,
clashScore=7.65,
overallScore=1.75,
protRSRefine=protRSRefine)
else:
self.checkRSRefineResults(ramOutliers=0.00,
ramFavored=94.42,
rotOutliers=11.04,
cbetaOutliers=0,
clashScore=12.92,
overallScore=2.79,
protRSRefine=protRSRefine)
# MolProbity2
args = {'inputStructure': protRSRefine.outputPdb, 'numberOfThreads': 4}
protMolProbity2 = self.newProtocol(PhenixProtRunMolprobity, **args)
protMolProbity2.setObjLabel('MolProbity validation\n'
'volume and pdb\nnucleosome')
self.launchProtocol(protMolProbity2)
# check MolProbity results
if Plugin.getPhenixVersion() == PHENIXVERSION:
self.checkMPResults(ramOutliers=0.00,
ramFavored=95.92,
rotOutliers=0.16,
cbetaOutliers=0,
clashScore=7.46,
overallScore=1.69,
protMolProbity=protMolProbity2)
elif Plugin.getPhenixVersion() == PHENIXVERSION18:
self.checkMPResults(ramOutliers=0.00,
ramFavored=96.19,
rotOutliers=12.32,
cbetaOutliers=0,
clashScore=12.45,
overallScore=2.69,
protMolProbity=protMolProbity2)
elif Plugin.getPhenixVersion() == PHENIXVERSION19:
self.checkMPResults(ramOutliers=0.00,
ramFavored=98.23,
rotOutliers=2.72,
cbetaOutliers=0,
clashScore=7.65,
overallScore=1.75,
protMolProbity=protMolProbity2)
else:
self.checkMPResults(ramOutliers=0.00,
ramFavored=94.42,
rotOutliers=12.96,
cbetaOutliers=0,
clashScore=12.96,
overallScore=2.80,
protMolProbity=protMolProbity2)
# TODO: Talk to Roberto if we have to continue testing these values (rotOutliers)
# validation_cryoEM
args = {'resolution': 4.0, 'inputStructure': protRSRefine.outputPdb}
protValCryoEM = self.newProtocol(PhenixProtRunValidationCryoEM, **args)
protValCryoEM.setObjLabel('ValCryoEM\nafter RSRefine\nnucleosome and '
'protRSRefine.outputPdb\n')
try:
self.launchProtocol(protValCryoEM)
except Exception as e:
self.assertTrue(
True, "This test should return a error message as '"
" Protocol has validation errors:\n"
"Binary '/usr/local/phenix-1.13-2998/modules/phenix/phenix/"
"command_line/validation_cryoem.py' does not exists.\n"
"Check if you need to upgrade your PHENIX version to run "
"VALIDATION_CRYOEM.\nYour current PHENIX version is 1.13.\n"
"Check configuration file: " + Config.SCIPION_LOCAL_CONFIG +
"\n"
"and set VALIDATION_CRYOEM and PHENIX_HOME variables properly.\n"
"Current values:\nPHENIX_HOME = /usr/local/phenix-1.13-2998\n")
return
# self.assertTrue(False)
# check validation cryoem results
if Plugin.getPhenixVersion() == PHENIXVERSION18:
self.checkValCryoEMResults(ramOutliers=0.00,
ramFavored=96.19,
rotOutliers=12.32,
cbetaOutliers=0,
clashScore=12.40,
overallScore=2.69,
protValCryoEM=protValCryoEM)
elif Plugin.getPhenixVersion() == PHENIXVERSION19:
self.checkValCryoEMResults(ramOutliers=0.00,
ramFavored=98.23,
rotOutliers=2.72,
cbetaOutliers=0,
clashScore=7.65,
overallScore=1.75,
protValCryoEM=protValCryoEM)
else:
self.checkValCryoEMResults(ramOutliers=0.00,
ramFavored=98.58,
rotOutliers=1.70,
cbetaOutliers=0,
clashScore=2.09,
overallScore=1.16,
protValCryoEM=protValCryoEM)
def testValCryoEMFromVolumeAndPDB1(self):
""" This test checks that phenix real_space_refine protocol runs
with a volume provided directly as inputVol, the input PDB was fitted
to the volume and refined previously by coot and refmac withouth mask
in another project; (MolProbity has been run to compare values before
and after refinement); default refine strategy;
MolProbity and ValCryoEM after RSRefine should show identical values
"""
print("Run phenix real_space_refine from imported volume and pdb file "
"previously fitted and refined by Coot and Refmac without mask "
"(MolProbity has been run to compare values before and after "
"refinement); default refine strategy; MolProbity and ValCryoEM "
"after RSRefine should show identical values")
# Import Volume
volume_refmac3 = self._importVolRefmac3()
# import PDB
structure_refmac3 = self._importStructRefmac3()
#MolProbity
args = {
'inputVolume': volume_refmac3,
'resolution': 3.5,
'inputStructure': structure_refmac3,
'numberOfThreads': 4
}
protMolProbity = self.newProtocol(PhenixProtRunMolprobity, **args)
protMolProbity.setObjLabel('MolProbity validation\n'
'volume and pdb\n')
self.launchProtocol(protMolProbity)
# check MolProbity results
self.checkMPResults(ramOutliers=0.47,
ramFavored=83.96,
rotOutliers=5.68,
cbetaOutliers=1,
clashScore=4.77,
overallScore=2.50,
protMolProbity=protMolProbity)
# real_space_refine
args = {
'inputVolume': volume_refmac3,
'resolution': 3.5,
'inputStructure': structure_refmac3
# default parameters in Optimization strategy options
}
if Plugin.getPhenixVersion() == PHENIXVERSION:
args['doSecondary'] = False
protRSRefine = self.newProtocol(PhenixProtRunRSRefine, **args)
protRSRefine.setObjLabel('RSRefine\n refmac3.mrc and ' 'refmac3.pdb\n')
self.launchProtocol(protRSRefine)
# check real_space_refine results
if Plugin.getPhenixVersion() == PHENIXVERSION:
self.checkRSRefineResults(ramOutliers=0.00,
ramFavored=95.75,
rotOutliers=0.00,
cbetaOutliers=0,
clashScore=2.09,
overallScore=1.27,
protRSRefine=protRSRefine)
else:
self.checkRSRefineResults(ramOutliers=0.00,
ramFavored=96.23,
rotOutliers=3.41,
cbetaOutliers=0,
clashScore=4.17,
overallScore=1.86,
protRSRefine=protRSRefine)
# MolProbity2
args = {
'inputVolume': volume_refmac3,
'resolution': 3.5,
'inputStructure': protRSRefine.outputPdb,
'numberOfThreads': 4
}
protMolProbity2 = self.newProtocol(PhenixProtRunMolprobity, **args)
protMolProbity2.setObjLabel('MolProbity\n' 'after RSRefine\n')
self.launchProtocol(protMolProbity2)
# check MolProbity results
if Plugin.getPhenixVersion() == PHENIXVERSION:
self.checkMPResults(ramOutliers=0.00,
ramFavored=95.75,
rotOutliers=0.00,
cbetaOutliers=0,
clashScore=2.09,
overallScore=1.27,
protMolProbity=protMolProbity2)
elif Plugin.getPhenixVersion() == PHENIXVERSION19:
self.checkMPResults(ramOutliers=0.00,
ramFavored=98.58,
rotOutliers=1.70,
cbetaOutliers=0,
clashScore=2.09,
overallScore=1.16,
protMolProbity=protMolProbity2)
else:
self.checkMPResults(ramOutliers=0.00,
ramFavored=96.23,
rotOutliers=3.41,
cbetaOutliers=0,
clashScore=4.17,
overallScore=1.86,
protMolProbity=protMolProbity2)
# validation_cryoEM
args = {
'inputVolume': volume_refmac3,
'resolution': 3.5,
'inputStructure': protRSRefine.outputPdb
}
protValCryoEM = self.newProtocol(PhenixProtRunValidationCryoEM, **args)
protValCryoEM.setObjLabel('ValCryoEM\nafter RSRefine\nrefmac3.mrc and '
'protRSRefine.outputPdb\n')
try:
self.launchProtocol(protValCryoEM)
except Exception as e:
self.assertTrue(
True, "This test should return a error message as '"
" Protocol has validation errors:\n"
"Binary '/usr/local/phenix-1.13-2998/modules/phenix/phenix/"
"command_line/validation_cryoem.py' does not exists.\n"
"Check if you need to upgrade your PHENIX version to run "
"VALIDATION_CRYOEM.\nYour current PHENIX version is 1.13.\n"
"Check configuration file: " + Config.SCIPION_LOCAL_CONFIG +
"\n"
"and set VALIDATION_CRYOEM and PHENIX_HOME variables properly.\n"
"Current values:\nPHENIX_HOME = /usr/local/phenix-1.13-2998\n")
return
# self.assertTrue(False)
# check validation cryoem results
self.checkValCryoEMResults(ramOutliers=0.00,
ramFavored=96.70,
rotOutliers=3.98,
cbetaOutliers=0,
clashScore=4.47,
overallScore=1.89,
protValCryoEM=protValCryoEM)
def createOutputStep(self):
# get emringer information
# values are stored in python pickle files
# string to be run by phenix python
# temporary file with emringer values
test = self.doTest
if test == True:
EMRINGERTRANSFERFILENAME = self._getExtraPath(
self.EMRINGERTRANSFERFILENAME)
else:
EMRINGERTRANSFERFILENAME = self._getTmpPath(
self.EMRINGERTRANSFERFILENAME)
# directory with files
plots = glob.glob(self._getExtraPath("*_plots"))[0]
# emringer main file
mainDataFile = glob.glob(self._getExtraPath("*_emringer.pkl"))[0]
# file with emringer score
EMRINGERSCORESFILENAME = os.path.join(plots,
self.EMRINGERSCORESFILENAME)
EMRINGERFILES = []
keyList = []
# files with other scores
EMRINGEROPTIMALTHRESHOLDFILENAME = \
os.path.join(plots, self.EMRINGEROPTIMALTHRESHOLDFILENAME)
EMRINGERFILES.append(EMRINGEROPTIMALTHRESHOLDFILENAME)
keyList.append('Optimal Threshold')
EMRINGERROTAMERRATIOSFILENAME = \
os.path.join(plots, self.EMRINGERROTAMERRATIOSFILENAME)
EMRINGERFILES.append(EMRINGERROTAMERRATIOSFILENAME)
keyList.append('Rotamer-Ratio')
EMRINGERMAXZSCOREFILENAME = \
os.path.join(plots, self.EMRINGERMAXZSCOREFILENAME)
EMRINGERFILES.append(EMRINGERMAXZSCOREFILENAME)
keyList.append('Max Zscore')
# I do not want to use the local pickle version
# but the version used to create the files
# that is the phenix.python version
command = """import pickle
import json
import sys
import collections
from mmtbx.ringer.em_scoring import parse_pickle
from mmtbx.ringer.em_rolling import easy_pickle, RingerDict
def pickleData(file):
with open(file,"r") as f:
return pickle.load(f)
# process file %s"
data = pickleData('%s')
dataDict = collections.OrderedDict()
# index to maxscore
maxScore = max(data)
maxScoreIndex = data.index(maxScore)
""" % (EMRINGERSCORESFILENAME, EMRINGERSCORESFILENAME)
for key, fileName in zip(keyList, EMRINGERFILES):
command += "# process file %s\n" % fileName
command += "data = pickleData('%s')\n" % fileName
command += "dataDict['%s'] = data[maxScoreIndex]\n" % key
command += """file_name='%s'
waves, thresholds = parse_pickle(file_name, out=sys.stdout)
# waves: list of ringer_residue objects (Non-gamma-branched, non-proline
# aminoacids with a non-H gamma atom) used in global EMRinger score
# computation
dataDict['Model Length'] = len(waves)
dataDict['EMRinger Score'] = maxScore # find max score
# dataDict is shown in a table
# except for those attributes starting with "_"
dataDict['_thresholds'] = thresholds
dataDict['_maxScoreIndex'] = maxScoreIndex
ringer_results = easy_pickle.load(file_name)
# ringer_results: list of ringer_residue objects (aminoacids with gamma
# carbon; at least one Chi angle)
hierarchy = RingerDict(ringer_results, 0)
chains = sorted(hierarchy.get_chains())
dataDict['_chains'] = chains
# Aminoacids that contain non-H gamma atom (at least one Chi angle; included in
# ringer_results)
dictResidue = {}
formatResidue = []
for residue in ringer_results:
resFormatTmp = residue.format()
formatResidue.append(resFormatTmp)
dictResidue[resFormatTmp] = ringer_results.index(residue)
dataDict['_residues_format'] = formatResidue
dataDict['_residues_dict'] = dictResidue
""" % mainDataFile
command += """with open('%s',"w") as f:
f.write(json.dumps(dataDict))
""" % (EMRINGERTRANSFERFILENAME)
pythonFileName = EMRINGERTRANSFERFILENAME.replace('.txt', '.py')
# write script file
with open(pythonFileName, "w") as f:
f.write(command)
# execute file with phenix.python
Plugin.runPhenixProgram("", pythonFileName)
# read file in scipion python
with open(EMRINGERTRANSFERFILENAME, "r") as f:
self.stringDataDict = String(f.read())
# self.dataDict = json.loads(f.read())
self._store()
def testPhenixRSRefineFromVolumeAndCIF9(self):
""" This test checks that phenix real_space_refine protocol runs
with a volume associated to the input mmCIF from the
data bank; alternative refine strategy; (MolProbity has been run
to compare values before and after refinement); outputFormat =
mmCIF.
"""
print("Run phenix real_space_refine from imported cif "
"file from data banks (vol origin 0.0, 0.0, 0.0) and "
"associated volume; "
"alternative refine strategy; (MolProbity has been run to "
"compare values before and after refinement); outputFormat = "
"mmCIF.")
# import PDB
structure_hemo_cif2 = self._importStructHemoCIF2()
# chimera operate to repair cif file
extraCommands = ""
extraCommands += "scipionwrite #3 " \
"prefix repaired_CIF_ChimeraX_\n"
extraCommands += "exit\n"
args = {
'extraCommands': extraCommands,
'pdbFileToBeRefined': structure_hemo_cif2
}
protChimera = self.newProtocol(ChimeraProtOperate, **args)
protChimera.setObjLabel('chimera operate\n repairing CIF\n')
self.launchProtocol(protChimera)
result = eval("protChimera.repaired_CIF_ChimeraX_Atom_struct__3_%06d" % \
protChimera.getObjId())
# MolProbity
args = {
'resolution': 3.2,
'inputStructure': result,
}
protMolProbity = self.newProtocol(PhenixProtRunMolprobity, **args)
protMolProbity.setObjLabel('MolProbity validation\n'
'cif and associated map\n')
self.launchProtocol(protMolProbity)
# check MolProbity results
self.checkMPResults(ramOutliers=0.00,
ramFavored=95.23,
rotOutliers=0.43,
cbetaOutliers=0,
clashScore=3.53,
overallScore=1.48,
protMolProbity=protMolProbity)
# real_space_refine
args = {
'resolution': 3.2,
'inputStructure': structure_hemo_cif2,
'outputFormat': mmCIF,
'localGridSearch': True,
'morphing': True,
'simulatedAnnealing': True,
'adp': False
}
if Plugin.getPhenixVersion() == PHENIXVERSION:
args['doSecondary'] = False
protRSRefine = self.newProtocol(PhenixProtRunRSRefine, **args)
protRSRefine.setObjLabel('RSRefine hemo\n'
'5ni1.cif and associated map\n'
'emd_3488.map\nalternative '
'refine strategy')
self.launchProtocol(protRSRefine)
# check real_space_refine results
if Plugin.getPhenixVersion() == PHENIXVERSION:
self.checkRSRefineResults(ramOutliers=0.00,
ramFavored=97.35,
rotOutliers=0.00,
cbetaOutliers=0,
clashScore=2.10,
overallScore=1.11,
protRSRefine=protRSRefine)
elif Plugin.getPhenixVersion() == PHENIXVERSION18:
self.checkRSRefineResults(ramOutliers=0.00,
ramFavored=97.52,
rotOutliers=0.00,
cbetaOutliers=0,
clashScore=3.64,
overallScore=1.25,
protRSRefine=protRSRefine)
else:
self.checkRSRefineResults(ramOutliers=0.00,
ramFavored=96.29,
rotOutliers=3.47,
cbetaOutliers=0,
clashScore=4.86,
overallScore=1.50,
protRSRefine=protRSRefine)
def _runChangingCifFormatSuperpose(self, list_args):
cwd = os.getcwd() + "/" + self._getExtraPath()
try:
if list_args[0].endswith(".cif") and list_args[1].endswith(".cif"):
try:
# upgrade cifs
list_args1 = []
for i in range(0, 2):
list_args1.append(fromCIFTommCIF(list_args[i], list_args[i]))
args1 = list_args1[0] + " " + list_args1[1]
Plugin.runPhenixProgram(Plugin.getProgram(SUPERPOSE), args1,
extraEnvDict=None, cwd=cwd)
except:
# convert cifs to pdbs
list_args2 = []
for i in range(0, 2):
list_args2.append(fromCIFToPDB(
list_args[i], list_args[i].replace('.cif', '.pdb')))
args2 = list_args2[0] + " " + list_args2[1]
Plugin.runPhenixProgram(Plugin.getProgram(SUPERPOSE), args2,
extraEnvDict=None, cwd=cwd)
elif list_args[0].endswith(".cif") and list_args[1].endswith(".pdb"):
try:
# pdbs: convert cif to pdb
list_args1 = []
list_args1.append(fromCIFToPDB(
list_args[0], list_args[0].replace('.cif', '.pdb')))
args1 = list_args1[0] + " " + list_args[1]
Plugin.runPhenixProgram(Plugin.getProgram(SUPERPOSE), args1,
extraEnvDict=None, cwd=cwd)
except:
try:
# cifs: convert pdb to cif
list_args2 = []
list_args2.append(fromPDBToCIF(
list_args[1], list_args[1].replace('.pdb', '.cif')))
args2 = list_args[0] + " " + list_args2[0]
Plugin.runPhenixProgram(Plugin.getProgram(SUPERPOSE), args2,
extraEnvDict=None, cwd=cwd)
except:
# upgrade cif
list_args3 = []
list_args0 = args2.split()
for i in range(0, 2):
list_args3[i].append(fromCIFTommCIF(
list_args0[i], list_args0[i]))
args3 = list_args3[0] + " " + list_args3[1]
Plugin.runPhenixProgram(Plugin.getProgram(SUPERPOSE),
args3, extraEnvDict=None, cwd=cwd)
elif list_args[0].endswith(".pdb") and list_args[1].endswith(".cif"):
try:
# pdbs: convert cif to pdb
list_args1 = []
list_args1.append(fromCIFToPDB(
list_args[1], list_args[1].replace('.cif', '.pdb')))
args1 = list_args[0] + " " + list_args1[0]
Plugin.runPhenixProgram(Plugin.getProgram(SUPERPOSE), args1,
extraEnvDict=None, cwd=cwd)
except:
try:
# cifs: convert pdb to cif
list_args2 = []
list_args2.append(fromPDBToCIF(
list_args[0], list_args[0].replace('.pdb', '.cif')))
args2 = list_args2[0] + " " + list_args[1]
Plugin.runPhenixProgram(Plugin.getProgram(SUPERPOSE), args2,
extraEnvDict=None, cwd=cwd)
except:
# upgrade cifs
list_args3 = []
list_args0 = args2.split()
for i in range(0, 2):
list_args3.append(fromCIFTommCIF(
list_args0[i], list_args0[i]))
args3 = list_args3[0] + " " + list_args3[1]
Plugin.runPhenixProgram(Plugin.getProgram(SUPERPOSE),
args3, extraEnvDict=None, cwd=cwd)
except:
# biopython conversion
aSH = AtomicStructHandler()
try:
for i in range(0, 2):
aSH.read(list_args[i])
aSH.write(list_args[i])
args = list_args[0] + " " + list_args[1]
Plugin.runPhenixProgram(Plugin.getProgram(SUPERPOSE),
args, extraEnvDict=None, cwd=cwd)
except:
print("CIF file standarization failed.")
def testValCryoEMFFromVolumeAndCIFFromPDB(self):
""" This test checks that phenix real_space_refine protocol runs
with a volume provided directly as inputVol (with half1 and half2)
and the input CIF from data banks; default refine strategy;
(MolProbity has been run to
compare values before and after refinement). MolProbity and
ValCryoEM after RSRefine should show identical values.
"""
print("Run phenix real_space_refine from imported volume and pdb file "
"from data banks (vol origin 0.0, 0.0, 0.0); default refine "
"strategy; (MolProbity has been run to compare values before "
"and after refinement). MolProbity and ValCryoEM after RSRefine "
"should show identical values.")
# Import Volume
volume_hemo_org = self._importVolHemoOrg()
# import cif
structure_hemo_cif_PDB = self._importStructHemoFromDB()
# chimera operate to repair cif file
extraCommands = ""
extraCommands += "scipionwrite #2 " \
"prefix repaired_CIF_ChimeraX_\n"
extraCommands += "exit\n"
args = {
'extraCommands': extraCommands,
'pdbFileToBeRefined': structure_hemo_cif_PDB
}
protChimera = self.newProtocol(ChimeraProtOperate, **args)
protChimera.setObjLabel('chimera operate\n repairing CIF\n')
self.launchProtocol(protChimera)
result = eval("protChimera.repaired_CIF_ChimeraX_Atom_struct__2_%06d" % \
protChimera.getObjId())
# MolProbity
args = {
'inputVolume': volume_hemo_org,
'resolution': 3.2,
'inputStructure': result,
'numberOfThreads': 4
}
protMolProbity = self.newProtocol(PhenixProtRunMolprobity, **args)
protMolProbity.setObjLabel('MolProbity validation\n'
'volume\nand cif from PDB\nhemoglobin')
self.launchProtocol(protMolProbity)
# check MolProbity results
self.checkMPResults(ramOutliers=0.00,
ramFavored=95.23,
rotOutliers=0.43,
cbetaOutliers=0,
clashScore=3.53,
overallScore=1.48,
protMolProbity=protMolProbity)
# real_space_refine
args = {
'inputVolume': volume_hemo_org,
'resolution': 3.2,
'inputStructure': structure_hemo_cif_PDB,
'numberOfThreads': 4
}
if Plugin.getPhenixVersion() == PHENIXVERSION:
args['doSecondary'] = False
protRSRefine = self.newProtocol(PhenixProtRunRSRefine, **args)
protRSRefine.setObjLabel('RSRefine hemo\n emd_3488.map\nand '
'5ni1.cif from PDB\n')
self.launchProtocol(protRSRefine)
# check real_space_refine results
if Plugin.getPhenixVersion() == PHENIXVERSION:
self.checkRSRefineResults(ramOutliers=0.00,
ramFavored=97.53,
rotOutliers=0.00,
cbetaOutliers=0,
clashScore=2.43,
overallScore=1.12,
protRSRefine=protRSRefine)
elif Plugin.getPhenixVersion() == PHENIXVERSION18:
self.checkRSRefineResults(ramOutliers=0.00,
ramFavored=96.64,
rotOutliers=4.77,
cbetaOutliers=0,
clashScore=6.07,
overallScore=2.06,
protRSRefine=protRSRefine)
elif Plugin.getPhenixVersion() == PHENIXVERSION19:
self.checkRSRefineResults(ramOutliers=0.00,
ramFavored=97.35,
rotOutliers=2.39,
cbetaOutliers=0,
clashScore=3.87,
overallScore=1.59,
protRSRefine=protRSRefine)
else:
# values obtained from phenix GUI v. 1.16
# (minimization_global + adp)
self.checkRSRefineResults(ramOutliers=0.00,
ramFavored=96.11,
rotOutliers=3.04,
cbetaOutliers=0,
clashScore=5.30,
overallScore=1.92,
protRSRefine=protRSRefine)
# MolProbity2
args = {
'inputVolume': volume_hemo_org,
'resolution': 3.2,
'inputStructure': protRSRefine.outputPdb,
'numberOfThreads': 4
}
protMolProbity2 = self.newProtocol(PhenixProtRunMolprobity, **args)
protMolProbity2.setObjLabel('MolProbity validation\n'
'volume\nand cif\nhemoglobin')
self.launchProtocol(protMolProbity2)
# check MolProbity results
if Plugin.getPhenixVersion() == PHENIXVERSION:
self.checkMPResults(ramOutliers=0.00,
ramFavored=97.53,
rotOutliers=0.00,
cbetaOutliers=0,
clashScore=2.43,
overallScore=1.12,
protMolProbity=protMolProbity2)
elif Plugin.getPhenixVersion() == PHENIXVERSION18:
self.checkMPResults(ramOutliers=0.00,
ramFavored=96.64,
rotOutliers=4.77,
cbetaOutliers=0,
clashScore=6.07,
overallScore=2.06,
protMolProbity=protMolProbity2)
elif Plugin.getPhenixVersion() == PHENIXVERSION19:
self.checkMPResults(ramOutliers=0.00,
ramFavored=97.35,
rotOutliers=2.39,
cbetaOutliers=0,
clashScore=3.87,
overallScore=1.59,
protMolProbity=protMolProbity2)
else:
self.checkMPResults(ramOutliers=0.00,
ramFavored=96.11,
rotOutliers=3.04,
cbetaOutliers=0,
clashScore=5.30,
overallScore=1.92,
protMolProbity=protMolProbity2)
# validation_cryoEM
args = {
'inputVolume': volume_hemo_org,
'resolution': 3.2,
'inputStructure': protRSRefine.outputPdb
}
protValCryoEM = self.newProtocol(PhenixProtRunValidationCryoEM, **args)
protValCryoEM.setObjLabel(
'ValCryoEM\nafter RSRefine\nvolume_hemo_org and '
'protRSRefine.outputPdb\n')
try:
self.launchProtocol(protValCryoEM)
except Exception as e:
self.assertTrue(
True, "This test should return a error message as '"
" Protocol has validation errors:\n"
"Binary '/usr/local/phenix-1.13-2998/modules/phenix/phenix/"
"command_line/validation_cryoem.py' does not exists.\n"
"Check if you need to upgrade your PHENIX version to run "
"VALIDATION_CRYOEM.\nYour current PHENIX version is 1.13.\n"
"Check configuration file: " + Config.SCIPION_LOCAL_CONFIG +
"\n"
"and set VALIDATION_CRYOEM and PHENIX_HOME variables properly.\n"
"Current values:\nPHENIX_HOME = /usr/local/phenix-1.13-2998\n")
return
# self.assertTrue(False)
# check validation cryoem results
if Plugin.getPhenixVersion() == PHENIXVERSION19:
self.checkValCryoEMResults(ramOutliers=0.00,
ramFavored=97.35,
rotOutliers=2.39,
cbetaOutliers=0,
clashScore=3.87,
overallScore=1.59,
protValCryoEM=protValCryoEM)
else:
self.checkValCryoEMResults(ramOutliers=0.00,
ramFavored=96.11,
rotOutliers=3.04,
cbetaOutliers=0,
clashScore=5.30,
overallScore=1.92,
protValCryoEM=protValCryoEM)
class PhenixProtRunRSRefine(PhenixProtRunRefinementBase):
"""Tool for extensive real-space refinement of an atomic structure
against the map provided. The map can be derived from X-ray or neutron
crystallography, or cryoEM. The program obtains a model that fits the map
as well as possible having appropriate geometry. The model should not show
validation outliers, such as Ramachandran plot or rotamer outliers.
"""
_label = 'real space refine'
_program = ""
# _version = VERSION_1_2
REALSPACEFILE = 'real_space.mrc'
if Plugin.getPhenixVersion() != PHENIXVERSION:
VALIDATIONCRYOEMPKLFILE = 'validation_cryoem.pkl'
# --------------------------- DEFINE param functions -------------------
def _defineParams(self, form):
super(PhenixProtRunRSRefine, self)._defineParams(form)
param = form.getParam('inputVolume')
param.help.set(
"\nSet the starting volume.\nPhenix will refine the "
"atomic structure according to the volume density.\n"
"Volume and atomic structure have to be correctly fitted. "
"Otherwise, values of real-space correlation will indicate "
"not correlation at all.\n")
form.addParam(
"doSecondary",
BooleanParam,
label="Secondary structure",
default=True,
expertLevel=LEVEL_ADVANCED,
help="Set to TRUE to use secondary structure "
"restraints.\nOnly for PHENIX versions higher than 1.13.")
form.addParam("macroCycles",
IntParam,
label="Macro cycles",
default=5,
expertLevel=LEVEL_ADVANCED,
help="Number of iterations of refinement.\nAlthough 5 "
"macro-cycles is usually sufficient, in cases in "
"which model geometry or/and model-to-map fit is "
"poor the use of more macro-cycles could be "
"helpful.\n")
group = form.addGroup('Optimization strategy options')
group.addParam('minimizationGlobal',
BooleanParam,
label="Global minimization: ",
default=True,
expertLevel=LEVEL_ADVANCED,
help="Phenix default parameter to look for the global "
"minimum of the model.\nGenerally, refinement "
"with all defaults is "
"sufficient.\nOther options "
"of use: run=minimization_global+local_grid_search"
"+morphing+simulated_annealing\n")
group.addParam('rigidBody',
BooleanParam,
label="Rigid body: ",
default=False,
expertLevel=LEVEL_ADVANCED,
help="Refinement strategy that considers groups of "
"atoms that move (rotate and translate) as a "
"single body.\n")
group.addParam('localGridSearch',
BooleanParam,
label="Local grid search: ",
default=False,
expertLevel=LEVEL_ADVANCED,
help="Refinement strategy that considers "
"local rotamer fitting.\n\n Generally, refinement "
"with all defaults is sufficient.\n Including "
"local fitting, morphing, "
"or simulated annealing "
"( local_grid_search+morphing+simulated_annealing) "
"into refinement may significantly increase "
"runtime.\nOther options "
"of use: run=minimization_global+local_grid_search"
"+morphing+simulated_annealing\n")
group.addParam('morphing',
BooleanParam,
label="Morphing ",
default=False,
expertLevel=LEVEL_ADVANCED,
help="Morphing procedure distorts a model to match an "
"electron density map.\n\nGenerally, refinement "
"with all defaults is "
"sufficient.\n Including local fitting, morphing, "
"or simulated annealing "
"( local_grid_search+morphing+simulated_annealing) "
"into refinement may significantly increase "
"runtime.\nOther options "
"of use: run=minimization_global+local_grid_search"
"+morphing+simulated_annealing\n")
group.addParam('simulatedAnnealing',
BooleanParam,
label="Simulated annealing ",
default=False,
expertLevel=LEVEL_ADVANCED,
help="Optimization technique known as molecular "
"dynamics refinement; it minimizes the energy of "
"the model.\n"
"Generally, refinement with all defaults is "
"sufficient.\n Including local fitting, morphing, "
"or simulated annealing "
"( local_grid_search+morphing+simulated_annealing) "
"into refinement may significantly increase "
"runtime.\nOther options "
"of use: run=minimization_global+local_grid_search"
"+morphing+simulated_annealing\n")
group.addParam('adp',
BooleanParam,
label="Atomic Displacement Parameters (ADPs) ",
default=True,
expertLevel=LEVEL_ADVANCED,
help="Phenix default parameter.\nGenerally, refinement "
"with all defaults is sufficient.\n\nADP ("
"B-factors) refinement against the map is "
"performed at the last macro-cycle only. ")
# form.addParallelSection(threads=1, mpi=0)
# --------------------------- INSERT steps functions ---------------
def _insertAllSteps(self):
self._insertFunctionStep('convertInputStep', self.REALSPACEFILE)
self._insertFunctionStep('runRSrefineStep', self.REALSPACEFILE)
self._insertFunctionStep('runMolprobityStep', self.REALSPACEFILE)
if Plugin.getPhenixVersion() != PHENIXVERSION:
self._insertFunctionStep('runValidationCryoEMStep',
self.REALSPACEFILE)
self._insertFunctionStep('createOutputStep')
# --------------------------- STEPS functions --------------------------
def runRSrefineStep(self, tmpMapFile):
atomStruct = os.path.abspath(self.inputStructure.get().getFileName())
vol = os.path.abspath(self._getExtraPath(tmpMapFile))
args = self._writeArgsRSR(atomStruct, vol)
cwd = os.getcwd() + "/" + self._getExtraPath()
retry(
Plugin.runPhenixProgram,
Plugin.getProgram(REALSPACEREFINE),
args,
# cwd=os.path.abspath(self._getExtraPath()),
cwd=cwd,
listAtomStruct=[atomStruct],
log=self._log)
self.refinedFile = False
for item in os.listdir(self._getExtraPath()):
p = re.compile('\d+')
if p.search(item) is not None and item.endswith(".cif"):
self.refinedFile = True
break
if self.refinedFile == False:
print("WARNING!!!\nPHENIX error:\n pdb_interpretation.clash_guard" \
" failure: High number of nonbonded interaction distances " \
"< 0.5. This error has been disable by running the same " \
"command with the same following additional " \
"argument:\npdb_interpretation.clash_guard." \
"nonbonded_distance_threshold=None ")
args += " pdb_interpretation.clash_guard." \
"nonbonded_distance_threshold=None"
retry(
Plugin.runPhenixProgram,
Plugin.getProgram(REALSPACEREFINE),
args,
# cwd=os.path.abspath(self._getExtraPath()),
cwd=cwd,
listAtomStruct=[atomStruct],
log=self._log)
def runMolprobityStep(self, tmpMapFile):
# PDBx/mmCIF
self._getRSRefineOutput()
atomStruct = os.path.abspath(self.outAtomStructName)
# starting volume (.mrc)
vol = os.path.abspath(self._getExtraPath(tmpMapFile))
args = self._writeArgsMolProbity(atomStruct, vol)
cwd = os.getcwd() + "/" + self._getExtraPath()
retry(
Plugin.runPhenixProgram,
Plugin.getProgram(MOLPROBITY2),
# args, cwd=os.path.abspath(self._getExtraPath()),
args,
cwd=cwd,
listAtomStruct=[atomStruct],
log=self._log)
def runValidationCryoEMStep(self, tmpMapFile):
# PDBx/mmCIF
atomStruct = os.path.abspath(self.outAtomStructName)
# starting volume (.mrc)
volume = os.path.abspath(self._getExtraPath(tmpMapFile))
if self.inputVolume.get() is not None:
vol = self.inputVolume.get()
else:
vol = self.inputStructure.get().getVolume()
args = self._writeArgsValCryoEM(atomStruct, volume, vol)
cwd = os.getcwd() + "/" + self._getExtraPath()
retry(
Plugin.runPhenixProgram,
Plugin.getProgram(VALIDATION_CRYOEM),
# args, cwd=os.path.abspath(self._getExtraPath()),
args,
cwd=cwd,
listAtomStruct=[atomStruct],
log=self._log)
def createOutputStep(self):
# self._getRSRefineOutput()
pdb = AtomStruct()
pdb.setFileName(self.outAtomStructName)
if self.inputVolume.get() is not None:
pdb.setVolume(self.inputVolume.get())
else:
pdb.setVolume(self.inputStructure.get().getVolume())
self._defineOutputs(outputPdb=pdb)
self._defineSourceRelation(self.inputStructure.get(), pdb)
if self.inputVolume.get() is not None:
self._defineSourceRelation(self.inputVolume.get(), pdb)
if Plugin.getPhenixVersion() == PHENIXVERSION:
MOLPROBITYOUTFILENAME = self._getExtraPath(
self.MOLPROBITYOUTFILENAME)
self._parseFile(MOLPROBITYOUTFILENAME)
else:
VALIDATIONCRYOEMPKLFILENAME = self._getExtraPath(
self.VALIDATIONCRYOEMPKLFILE)
self._readValidationPklFile(VALIDATIONCRYOEMPKLFILENAME)
self._store()
# --------------------------- INFO functions ---------------------------
def _validate(self):
errors = self.validateBase(REALSPACEREFINE, 'REALSPACEREFINE')
# Check that the input volume exist
if self._getInputVolume() is None:
errors.append("Error: You should provide a volume.\n")
return errors
def _citations(self):
return ['Barad_2015']
def _summary(self):
summary = PhenixProtRunRefinementBase._summary(self)
summary.append("https://www.phenix-online.org/documentation/reference/"
"real_space_refine.html")
return summary
# --------------------------- UTILS functions --------------------------
def _getRSRefineOutput(self):
inPdbName = os.path.basename(self.inputStructure.get().getFileName())
list_cif = []
for item in os.listdir(self._getExtraPath()):
p = re.compile('\d+')
if p.search(item) is not None and item.endswith(".cif"):
list_cif.append(item)
name = sorted(list_cif)[-1]
outAtomStructName = self._getExtraPath(name)
# convert cif to mmcif by using maxit program
# to get the right number and name of chains
log = self._log
self.outAtomStructName = outAtomStructName
fromCIFTommCIF(outAtomStructName, self.outAtomStructName, log)
def _writeArgsRSR(self, atomStruct, vol):
if Plugin.getPhenixVersion() == PHENIXVERSION19:
args = " "
else:
args = " model_file="
args += "%s " % atomStruct
if Plugin.getPhenixVersion() == PHENIXVERSION19:
args += " "
else:
args += " map_file="
args += "%s " % vol
args += " resolution=%f" % self.resolution
args += " secondary_structure.enabled=%s" % self.doSecondary
args += " run="
if self.minimizationGlobal == True:
args += "minimization_global+"
if self.rigidBody == True:
args += "rigid_body+"
if self.localGridSearch == True:
args += "local_grid_search+"
if self.morphing == True:
args += "morphing+"
if self.simulatedAnnealing == True:
args += "simulated_annealing+"
if self.adp == True:
args += "adp+"
args = args[:-1]
# args += " run=minimization_global+local_grid_search+morphing+simulated_annealing"
args += " macro_cycles=%d" % self.macroCycles
args += " model_format=pdb+mmcif"
# args += " write_pkl_stats=True"
args += " %s " % self.extraParams.get()
numberOfThreads = self.numberOfThreads.get()
if numberOfThreads > 1:
args += " nproc=%d" % numberOfThreads
return args
