def __init__(self,initial_model_directory,xce_logfile,datasource):
QtCore.QThread.__init__(self)
self.xce_logfile=xce_logfile
self.Logfile=XChemLog.updateLog(xce_logfile)
self.initial_model_directory=initial_model_directory
self.datasource=datasource
self.db=XChemDB.data_source(datasource)
def __init__(self, htmlDir, projectDir, database, xce_logfile):
self.htmlDir = htmlDir
self.projectDir = projectDir
self.Logfile = XChemLog.updateLog(xce_logfile)
self.db = XChemDB.data_source(database)
self.db_dict = None
self.pdb = None
def __init__(self,panddas_directory,datasource,initial_model_directory,xce_logfile,update_datasource_only,which_models):
QtCore.QThread.__init__(self)
self.panddas_directory=panddas_directory
self.datasource=datasource
self.initial_model_directory=initial_model_directory
self.db=XChemDB.data_source(self.datasource)
self.db.create_missing_columns()
self.db_list=self.db.get_empty_db_dict()
self.external_software=XChemUtils.external_software(xce_logfile).check()
self.xce_logfile=xce_logfile
self.Logfile=XChemLog.updateLog(xce_logfile)
self.update_datasource_only=update_datasource_only
self.which_models=which_models
self.already_exported_models=[]
self.RefmacParams={ 'HKLIN': '', 'HKLOUT': '',
'XYZIN': '', 'XYZOUT': '',
'LIBIN': '', 'LIBOUT': '',
'TLSIN': '', 'TLSOUT': '',
'TLSADD': '',
'NCYCLES': '10',
'MATRIX_WEIGHT': 'AUTO',
'BREF': ' bref ISOT\n',
'TLS': '',
'NCS': '',
'TWIN': '' }
def __init__(self, data_source_file,parent = None):
super(select_columns_to_show, self).__init__(parent)
self.columns_in_data_source=XChemDB.data_source(data_source_file).return_column_list()
self.column_dict={}
layout = QtGui.QVBoxLayout(self)
number_of_entries=len(self.columns_in_data_source)
columns_shown_in_dialog_column=25
grid = QtGui.QGridLayout()
x=0
y=0
columns_to_ignore=['Sample ID','ID']
for entries_added in range(number_of_entries):
if not self.columns_in_data_source[entries_added][1] in columns_to_ignore:
data_source_column = QtGui.QCheckBox(self.columns_in_data_source[entries_added][1])
self.column_dict[entries_added]=data_source_column
# data_source_column.toggle()
grid.addWidget(data_source_column, y,x)
y+=1
if y==columns_shown_in_dialog_column:
y=0
x+=1
layout.addLayout(grid)
# OK and Cancel buttons
buttons = QtGui.QDialogButtonBox(
QtGui.QDialogButtonBox.Ok | QtGui.QDialogButtonBox.Cancel,
QtCore.Qt.Horizontal, self)
buttons.accepted.connect(self.accept)
buttons.rejected.connect(self.reject)
layout.addWidget(buttons)
def update_data_source(db_file, crystalString, status):
db = XChemDB.data_source(db_file)
print "update mainTable set PANDDAStatus = '{0!s}' where CrystalName in ({1!s})".format(
status, "'" + crystalString.replace(",", "','") + "'")
db.execute_statement(
"update mainTable set PANDDAStatus = '{0!s}' where CrystalName in ({1!s})"
.format(status, "'" + crystalString.replace(",", "','") + "'"))
def get_datasource_summary(db_file):
db=XChemDB.data_source(db_file)
out_dict={}
out_dict['nr_samples']=len(db.execute_statement("select CrystalName from mainTable where CrystalName is not NULL;"))
out_dict['nr_samples_failed_to_mount']=len(db.execute_statement("select HarvestStatus from mainTable where HarvestStatus is 'fail';"))
out_dict['nr_smiles_for_samples']=len(db.execute_statement("select compoundSMILES from mainTable where compoundSMILES is not (NULL or '')"))
out_dict['nr_data_collection_success']=len(db.execute_statement("select DataCollectionOutcome from mainTable where DataCollectionOutcome is 'success';"))
out_dict['nr_data_collection_centring_fail']=len(db.execute_statement("select DataCollectionOutcome from mainTable where DataCollectionOutcome is 'Failed - centring failed';"))
out_dict['nr_data_collection_no-diffraction']=len(db.execute_statement("select DataCollectionOutcome from mainTable where DataCollectionOutcome is 'Failed - no diffraction';"))
out_dict['nr_data_collection_processing_fail']=len(db.execute_statement("select DataCollectionOutcome from mainTable where DataCollectionOutcome is 'Failed - processing';"))
out_dict['nr_data_collection_loop-empty']=len(db.execute_statement("select DataCollectionOutcome from mainTable where DataCollectionOutcome is 'Failed - loop empty';"))
out_dict['nr_data_collection_loop-broken']=len(db.execute_statement("select DataCollectionOutcome from mainTable where DataCollectionOutcome is 'Failed - loop broken';"))
out_dict['nr_data_collection_low-resolution']=len(db.execute_statement("select DataCollectionOutcome from mainTable where DataCollectionOutcome is 'Failed - low resolution';"))
out_dict['nr_data_collection_no-X-rays']=len(db.execute_statement("select DataCollectionOutcome from mainTable where DataCollectionOutcome is 'Failed - no X-rays';"))
out_dict['nr_data_collection_unknown']=len(db.execute_statement("select DataCollectionOutcome from mainTable where DataCollectionOutcome is 'Failed - unknown';"))
out_dict['nr_data_collection_failed']= out_dict['nr_data_collection_centring_fail'] + \
out_dict['nr_data_collection_no-diffraction'] + \
out_dict['nr_data_collection_processing_fail'] + \
out_dict['nr_data_collection_loop-empty'] + \
out_dict['nr_data_collection_loop-broken'] + \
out_dict['nr_data_collection_low-resolution'] + \
out_dict['nr_data_collection_no-X-rays'] + \
out_dict['nr_data_collection_unknown']
out_dict['nr_data_collection_pending']=out_dict['nr_samples'] - out_dict['nr_data_collection_success'] - \
out_dict['nr_data_collection_centring_fail'] - \
out_dict['nr_data_collection_no-diffraction'] - \
out_dict['nr_data_collection_processing_fail'] - \
out_dict['nr_data_collection_loop-empty'] - \
out_dict['nr_data_collection_loop-broken'] - \
out_dict['nr_data_collection_low-resolution'] - \
out_dict['nr_data_collection_no-X-rays'] - \
out_dict['nr_data_collection_unknown']
out_dict['nr_initial_maps_available']=len(db.execute_statement("select DimplePathToMTZ from mainTable where DimplePathToMTZ is not '';"))
out_dict['nr_initial_maps_fail']=len(db.execute_statement("select DataProcessingDimpleSuccessful from mainTable where DataProcessingDimpleSuccessful = 'False';"))
out_dict['nr_initial_maps_pending']=out_dict['nr_data_collection_success']-out_dict['nr_initial_maps_available']-out_dict['nr_initial_maps_fail']
out_dict['nr_pandda_hits']=len(db.execute_statement("select DimplePANDDAhit from mainTable where DimplePANDDAhit = 'True';"))
out_dict['nr_pandda_reject']=len(db.execute_statement("select DimplePANDDAreject from mainTable where DimplePANDDAreject = 'True';"))
out_dict['nr_pandda_processed']=len(db.execute_statement("select DimplePANDDAwasRun from mainTable where DimplePANDDAwasRun = 'True';"))-out_dict['nr_pandda_hits']-out_dict['nr_pandda_reject']
out_dict['nr_pandda_pending']=out_dict['nr_initial_maps_available']-out_dict['nr_pandda_hits']-out_dict['nr_pandda_reject']-out_dict['nr_pandda_processed']
out_dict['nr_cif_files']=len(db.execute_statement("select RefinementCIF from mainTable where RefinementCIF is not (Null or '');"))
out_dict['nr_analysis-pending']=len(db.execute_statement("select RefinementOutcome from mainTable where RefinementOutcome is '1 - Analysis Pending';"))
out_dict['nr_pandda-models']=len(db.execute_statement("select RefinementOutcome from mainTable where RefinementOutcome is '2 - PANDDA model';"))
out_dict['nr_in-refinement']=len(db.execute_statement("select RefinementOutcome from mainTable where RefinementOutcome is '3 - In Refinement';"))
out_dict['nr_comp-chem-ready']=len(db.execute_statement("select RefinementOutcome from mainTable where RefinementOutcome is '4 - ComChem ready';"))
out_dict['nr_deposition-ready']=len(db.execute_statement("select RefinementOutcome from mainTable where RefinementOutcome is '5 - Deposition ready';"))
return out_dict
def __init__(self,initial_model_directory,pandda_params,xce_logfile,datasource,run_pandda_analyse):
QtCore.QThread.__init__(self)
self.panddas_directory=pandda_params['out_dir']
self.pdb_style=pandda_params['pdb_style']
self.mtz_style=pandda_params['mtz_style']
self.Logfile=XChemLog.updateLog(xce_logfile)
self.initial_model_directory=initial_model_directory
self.db=XChemDB.data_source(datasource)
self.run_pandda_analyse=run_pandda_analyse
def change_links_to_selected_data_collection_outcome(sample,data_collection_dict,data_collection_column_three_dict,dataset_outcome_dict,initial_model_directory,data_source_file,xce_logfile):
Logfile=XChemLog.updateLog(xce_logfile)
# find out which row was selected in respective data collection table
selected_processing_result='n/a'
indexes=data_collection_column_three_dict[sample][0].selectionModel().selectedRows()
if indexes != []: # i.e. logfile exists
for index in sorted(indexes):
selected_processing_result=index.row()
for n,entry in enumerate(data_collection_dict[sample]):
if entry[0]=='logfile':
if entry[7]==selected_processing_result:
visit=entry[1]
run=entry[2]
autoproc=entry[4]
db_dict=entry[6]
outcome=dataset_outcome_dict[sample]
path_to_logfile=db_dict['DataProcessingPathToLogfile']
path_to_mtzfile=db_dict['DataProcessingPathToMTZfile']
mtz_filename=db_dict['DataProcessingMTZfileName']
log_filename=db_dict['DataProcessingLOGfileName']
relative_path_to_mtzfile='./'+path_to_mtzfile.replace(initial_model_directory,'')
if relative_path_to_mtzfile.startswith('.//'):
relative_path_to_mtzfile=relative_path_to_mtzfile.replace('.//','./')
relative_path_to_logfile='./'+path_to_logfile.replace(initial_model_directory,'')
if relative_path_to_logfile.startswith('.//'):
relative_path_to_logfile=relative_path_to_logfile.replace('.//','./')
# first check if folders and files exist
# since user might do this before data are actually copied over
if os.path.isdir(os.path.join(initial_model_directory,sample,'autoprocessing',visit+'-'+run+autoproc)):
db_dict['DataProcessingAutoAssigned']='False'
Logfile.insert('changing directory to: '+os.path.join(initial_model_directory,sample))
os.chdir(os.path.join(initial_model_directory,sample))
# first remove old links
os.system('/bin/rm '+sample+'.mtz 2> /dev/null')
os.system('/bin/rm '+sample+'.log 2> /dev/null')
# make new links
# Logfile.insert('setting symlink: '+os.path.join(path_to_logfile,log_filename)+' -> '+sample+'.log')
# os.symlink(os.path.join(path_to_logfile,log_filename),sample+'.log')
# Logfile.insert('setting symlink: '+os.path.join(path_to_mtzfile,mtz_filename)+' -> '+sample+'.mtz')
# os.symlink(os.path.join(path_to_mtzfile,mtz_filename),sample+'.mtz')
Logfile.insert('setting relative symlink: '+os.path.join(relative_path_to_logfile,log_filename)+' -> '+sample+'.log')
os.symlink(os.path.join(relative_path_to_logfile,log_filename),sample+'.log')
Logfile.insert('setting relative symlink: '+os.path.join(relative_path_to_mtzfile,mtz_filename)+' -> '+sample+'.mtz')
os.symlink(os.path.join(relative_path_to_mtzfile,mtz_filename),sample+'.mtz')
# update data source
data_source=XChemDB.data_source(data_source_file)
data_source.update_insert_data_source(sample,db_dict)
else:
Logfile.insert('please copy data to PROJECT DIRECTORY first!')
def __init__(self,pandda_params,xce_logfile,datasource):
self.data_directory=pandda_params['data_dir']
self.panddas_directory=pandda_params['out_dir']
self.min_build_datasets=pandda_params['min_build_datasets']
self.pdb_style=pandda_params['pdb_style']
self.mtz_style=pandda_params['mtz_style']
self.input_dir_structure=pandda_params['pandda_dir_structure']
self.problem_found=False
self.error_code=-1
self.Logfile=XChemLog.updateLog(xce_logfile)
self.db=XChemDB.data_source(datasource)
def __init__(self,initial_model_directory,run_dict,protocol,spg,ref,reso_limit,cc_half,xce_logfile,external_software,ccp4_scratch_directory,max_queue_jobs,database):
QtCore.QThread.__init__(self)
self.initial_model_directory=initial_model_directory
self.run_dict=run_dict
self.protocol=protocol
self.spg=spg
self.ref=ref
self.reso_limit=reso_limit
self.cc_half=cc_half
self.xce_logfile=xce_logfile
self.Logfile=XChemLog.updateLog(xce_logfile)
self.external_software=external_software
self.ccp4_scratch_directory=ccp4_scratch_directory
self.max_queue_jobs=max_queue_jobs
self.database=database
self.db=XChemDB.data_source(database)
def __init__(self,initial_model_directory,run_dict,protocol,spg,ref,reso_limit,cc_half,xce_logfile,external_software,ccp4_scratch_directory,max_queue_jobs,database,overwrite):
QtCore.QThread.__init__(self)
self.initial_model_directory=initial_model_directory
self.run_dict=run_dict
self.protocol=protocol
self.spg=spg
self.ref=ref
self.reso_limit=reso_limit
self.cc_half=cc_half
self.xce_logfile=xce_logfile
self.Logfile=XChemLog.updateLog(xce_logfile)
self.external_software=external_software
self.ccp4_scratch_directory=ccp4_scratch_directory
self.max_queue_jobs=max_queue_jobs
self.database=database
self.db=XChemDB.data_source(database)
self.overwrite=overwrite
def __init__(self,project_directory,xtalID,event_map,ligand_pdb,xce_logfile,db_file,resolution):
self.Logfile=XChemLog.updateLog(xce_logfile)
self.event_map=event_map
if not os.path.isfile(self.event_map):
self.Logfile.insert('cannot find Event map: '+self.event_map)
self.Logfile.insert('cannot convert event_map to structure factors!')
return None
self.project_directory=project_directory
self.xtalID=xtalID
self.event_map=event_map
self.ligand_pdb=ligand_pdb
self.event=event_map[event_map.rfind('/')+1:].replace('.map','').replace('.ccp4','')
# self.resolution=resolution
self.db=XChemDB.data_source(db_file)
# self.db_file=db_file
self.resolution=resolution
def __init__(self,pandda_params,xce_logfile,dataset_list,datasource):
QtCore.QThread.__init__(self)
self.data_directory=pandda_params['data_dir']
self.panddas_directory=pandda_params['out_dir']
self.submit_mode=pandda_params['submit_mode']
# if self.submit_mode == 'local machine':
# self.nproc=pandda_params['nproc']
# else:
# self.nproc='7'
self.nproc=pandda_params['nproc']
self.min_build_datasets=pandda_params['min_build_datasets']
self.pdb_style=pandda_params['pdb_style']
self.mtz_style=pandda_params['mtz_style']
self.sort_event=pandda_params['sort_event']
self.number_of_datasets=pandda_params['N_datasets']
self.max_new_datasets=pandda_params['max_new_datasets']
self.grid_spacing=pandda_params['grid_spacing']
self.filter_pdb=pandda_params['filter_pdb']
self.Logfile=XChemLog.updateLog(xce_logfile)
self.dataset_list=dataset_list
self.datasource=datasource
self.db=XChemDB.data_source(datasource)
for cif in glob.glob(os.path.join(compoundID + '_*.cif')):
foundCIF = True
allCIF += cif + ';'
print '---', cif, '===', allCIF
if foundCIF:
db_dict = {}
db_dict['CompoundStereo'] = 'TRUE'
db_dict['CompoundStereoSMILES'] = stereosmiles
db_dict['CompoundStereoCIFprogram'] = 'phenix.elbow'
db_dict['CompoundStereoCIFs'] = allCIF[:-1]
print '>>>', db_dict
updateDB(db, db_dict, xtal)
def updateDB(db, db_dict, xtal):
# update stereo field
# update restraintsprogram field
# update stereosmiles
db.update_data_source(xtal, db_dict)
if __name__ == '__main__':
compoundID = sys.argv[1]
sampleDir = sys.argv[2]
xtal = sampleDir[sampleDir.rfind('/') + 1:]
print 'sampleID:', xtal
database = sys.argv[3]
print 'database:', database
db = XChemDB.data_source(database)
enumerateStereoChem(compoundID, sampleDir, db, xtal)
def update_data_source(db_file, xtal, db_column, status):
db = XChemDB.data_source(db_file)
db_dict = {}
db_dict[db_column] = status
db.update_data_source(xtal, db_dict)
# last edited: 22/11/2016, 15:00
import os,sys
sys.path.append(os.path.join(os.getenv('XChemExplorer_DIR'),'lib'))
from XChemUtils import parse
import XChemDB
if __name__=='__main__':
db_file=sys.argv[1]
xtal=sys.argv[2]
initial_model_directory=sys.argv[3]
compoundID=sys.argv[4]
db=XChemDB.data_source(db_file)
db_dict={}
if os.path.isfile(os.path.join(initial_model_directory,xtal,'compound',compoundID+'.cif')) and \
os.path.getsize(os.path.join(initial_model_directory,xtal,'compound',compoundID+'.cif')) > 20 :
db_dict['RefinementCIF']=os.path.join(initial_model_directory,xtal,'compound',compoundID+'.cif')
db_dict['RefinementCIFStatus']='restraints generated'
else:
db_dict['RefinementCIF']=''
db_dict['RefinementCIFStatus']='restraints failed'
os.system('/bin/rm '+os.path.join(initial_model_directory,xtal,compoundID.replace(' ','')+'.pdb'))
os.system('/bin/rm '+os.path.join(initial_model_directory,xtal,compoundID.replace(' ','')+'.cif'))
os.system('/bin/rm '+os.path.join(initial_model_directory,xtal,compoundID.replace(' ','')+'.png'))
db.update_data_source(xtal,db_dict)
def update_data_source(db_file,crystalString,status):
db=XChemDB.data_source(db_file)
print "update mainTable set PANDDAStatus = '%s' where CrystalName in (%s)" %(status,"'"+crystalString.replace(",","','")+"'")
db.execute_statement("update mainTable set PANDDAStatus = '%s' where CrystalName in (%s)" %(status,"'"+crystalString.replace(",","','")+"'"))
"update panddaTable set RefinementOutcome = '3 - In Refinement' where CrystalName is '{0!s}' "
.format(xtal) +
"and (RefinementOutcome is null or RefinementOutcome is '1 - Analysis Pending' or RefinementOutcome is '2 - PANDDA model')"
)
db.execute_statement(sqlite)
if __name__ == '__main__':
print 'hallo'
db_file = sys.argv[1]
xtal = sys.argv[2]
inital_model_directory = sys.argv[3]
refinement_directory = sys.argv[4]
db = XChemDB.data_source(db_file)
db_dict = {}
db_dict = parse_pdb(inital_model_directory, xtal, db_dict)
db_dict = parse_mtz(inital_model_directory, xtal, db_dict)
db_dict = check_refmac_matrix_weight(refinement_directory, db_dict)
db_dict = parse_molprobity_output(inital_model_directory, xtal, db_dict)
db_dict = check_refmac_logfile(refinement_directory, db_dict)
update_ligand_information_in_panddaTable(inital_model_directory, xtal)
parse_ligand_validation(inital_model_directory, refinement_directory, xtal)
update_data_source(db_dict)
def __init__(self):
###########################################################################################
# read in settings file from XChemExplorer to set the relevant paths
print 'current dir',os.getcwd()
self.settings = pickle.load(open(".xce_settings.pkl","rb"))
print 'setting',self.settings
# self.refine_model_directory=self.settings['refine_model_directory']
self.database_directory=self.settings['database_directory']
self.xce_logfile=self.settings['xce_logfile']
self.data_source=self.settings['data_source']
self.db=XChemDB.data_source(self.data_source)
# checking for external software packages
self.external_software=XChemUtils.external_software(self.xce_logfile).check()
self.selection_criteria = [ '0 - All Datasets',
'1 - Analysis Pending',
'2 - PANDDA model',
'3 - In Refinement',
'4 - CompChem ready',
'5 - Deposition ready',
'6 - Deposited' ]
self.experiment_stage = [ ['Review PANDDA export', '2 - PANDDA model', 65000, 0, 0],
['In Refinement', '3 - In Refinement', 65000, 0, 0],
['Comp Chem Ready!', '4 - CompChem ready', 65000, 0, 0],
['Ready for Deposition!', '5 - Deposition ready', 65000, 0, 0] ]
self.ligand_confidence_category = [ '0 - no ligand present',
'1 - Low Confidence',
'2 - Correct ligand, weak density',
'3 - Clear density, unexpected ligand',
'4 - High Confidence' ]
self.ligand_site_information = self.db.get_list_of_pandda_sites_for_coot()
# this decides which samples will be looked at
self.selection_mode = ''
self.selected_site=''
# the Folder is kind of a legacy thing because my inital idea was to have separate folders
# for Data Processing and Refinement
self.project_directory = self.settings['initial_model_directory']
self.Serial=0
self.Refine=None
self.index = -1
self.Todo=[]
self.xtalID=''
self.compoundID=''
self.spider_plot=''
self.ligand_confidence=''
self.refinement_folder=''
# self.datasetOutcome=''
self.pdb_style='refine.pdb'
self.mtz_style='refine.mtz'
# stores imol of currently loaded molecules and maps
self.mol_dict = { 'protein': -1,
'ligand': -1,
'2fofc': -1,
'fofc': -1,
'event': -1 }
# two dictionaries which are flushed when a new crystal is loaded
# and which contain information to update the data source if necessary
self.db_dict_mainTable={}
self.db_dict_panddaTable={}
###########################################################################################
# some COOT settings
coot.set_map_radius(15)
coot.set_colour_map_rotation_for_map(0)
coot.set_colour_map_rotation_on_read_pdb_flag(0)
self.QualityIndicators = { 'RefinementRcryst': '-',
'RefinementRfree': '-',
'RefinementRfreeTraficLight': 'gray',
'RefinementResolution': '-',
'RefinementResolutionTL': 'gray',
'RefinementMolProbityScore': '-',
'RefinementMolProbityScoreTL': 'gray',
'RefinementRamachandranOutliers': '-',
'RefinementRamachandranOutliersTL': 'gray',
'RefinementRamachandranFavored': '-',
'RefinementRamachandranFavoredTL': 'gray',
'RefinementRmsdBonds': '-',
'RefinementRmsdBondsTL': 'gray',
'RefinementRmsdAngles': '-',
'RefinementRmsdAnglesTL': 'gray',
'RefinementMatrixWeight': '-' }
self.spider_plot_data = { 'PANDDA_site_ligand_id': '-',
'PANDDA_site_occupancy': '-',
'PANDDA_site_B_average': '-',
'PANDDA_site_B_ratio_residue_surroundings': '-',
'PANDDA_site_RSCC': '-',
'PANDDA_site_rmsd': '-',
'PANDDA_site_RSR': '-',
'PANDDA_site_RSZD': '-' }
# default refmac parameters
self.RefmacParams={ 'HKLIN': '', 'HKLOUT': '',
'XYZIN': '', 'XYZOUT': '',
'LIBIN': '', 'LIBOUT': '',
'TLSIN': '', 'TLSOUT': '',
'TLSADD': '',
'NCYCLES': '10',
'MATRIX_WEIGHT': 'AUTO',
'BREF': ' bref ISOT\n',
'TLS': '',
'NCS': '',
'TWIN': '' }
def get_datasource_summary(db_file):
db = XChemDB.data_source(db_file)
out_dict = {}
out_dict['nr_samples'] = len(
db.execute_statement(
"select CrystalName from mainTable where CrystalName is not NULL;")
)
out_dict['nr_samples_failed_to_mount'] = len(
db.execute_statement(
"select HarvestStatus from mainTable where HarvestStatus is 'fail';"
))
out_dict['nr_smiles_for_samples'] = len(
db.execute_statement(
"select compoundSMILES from mainTable where compoundSMILES is not (NULL or '')"
))
out_dict['nr_data_collection_success'] = len(
db.execute_statement(
"select DataCollectionOutcome from mainTable where DataCollectionOutcome is 'success';"
))
out_dict['nr_data_collection_centring_fail'] = len(
db.execute_statement(
"select DataCollectionOutcome from mainTable where DataCollectionOutcome is 'Failed - centring failed';"
))
out_dict['nr_data_collection_no-diffraction'] = len(
db.execute_statement(
"select DataCollectionOutcome from mainTable where DataCollectionOutcome is 'Failed - no diffraction';"
))
out_dict['nr_data_collection_processing_fail'] = len(
db.execute_statement(
"select DataCollectionOutcome from mainTable where DataCollectionOutcome is 'Failed - processing';"
))
out_dict['nr_data_collection_loop-empty'] = len(
db.execute_statement(
"select DataCollectionOutcome from mainTable where DataCollectionOutcome is 'Failed - loop empty';"
))
out_dict['nr_data_collection_loop-broken'] = len(
db.execute_statement(
"select DataCollectionOutcome from mainTable where DataCollectionOutcome is 'Failed - loop broken';"
))
out_dict['nr_data_collection_low-resolution'] = len(
db.execute_statement(
"select DataCollectionOutcome from mainTable where DataCollectionOutcome is 'Failed - low resolution';"
))
out_dict['nr_data_collection_no-X-rays'] = len(
db.execute_statement(
"select DataCollectionOutcome from mainTable where DataCollectionOutcome is 'Failed - no X-rays';"
))
out_dict['nr_data_collection_unknown'] = len(
db.execute_statement(
"select DataCollectionOutcome from mainTable where DataCollectionOutcome is 'Failed - unknown';"
))
out_dict['nr_data_collection_failed']= out_dict['nr_data_collection_centring_fail'] + \
out_dict['nr_data_collection_no-diffraction'] + \
out_dict['nr_data_collection_processing_fail'] + \
out_dict['nr_data_collection_loop-empty'] + \
out_dict['nr_data_collection_loop-broken'] + \
out_dict['nr_data_collection_low-resolution'] + \
out_dict['nr_data_collection_no-X-rays'] + \
out_dict['nr_data_collection_unknown']
out_dict['nr_data_collection_pending']=out_dict['nr_samples'] - out_dict['nr_data_collection_success'] - \
out_dict['nr_data_collection_centring_fail'] - \
out_dict['nr_data_collection_no-diffraction'] - \
out_dict['nr_data_collection_processing_fail'] - \
out_dict['nr_data_collection_loop-empty'] - \
out_dict['nr_data_collection_loop-broken'] - \
out_dict['nr_data_collection_low-resolution'] - \
out_dict['nr_data_collection_no-X-rays'] - \
out_dict['nr_data_collection_unknown']
out_dict['nr_initial_maps_available'] = len(
db.execute_statement(
"select DimplePathToMTZ from mainTable where DimplePathToMTZ is not '';"
))
out_dict['nr_initial_maps_fail'] = len(
db.execute_statement(
"select DataProcessingDimpleSuccessful from mainTable where DataProcessingDimpleSuccessful = 'False';"
))
out_dict['nr_initial_maps_pending'] = out_dict[
'nr_data_collection_success'] - out_dict[
'nr_initial_maps_available'] - out_dict['nr_initial_maps_fail']
out_dict['nr_pandda_hits'] = len(
db.execute_statement(
"select DimplePANDDAhit from mainTable where DimplePANDDAhit = 'True';"
))
out_dict['nr_pandda_reject'] = len(
db.execute_statement(
"select DimplePANDDAreject from mainTable where DimplePANDDAreject = 'True';"
))
out_dict['nr_pandda_processed'] = len(
db.execute_statement(
"select DimplePANDDAwasRun from mainTable where DimplePANDDAwasRun = 'True';"
)) - out_dict['nr_pandda_hits'] - out_dict['nr_pandda_reject']
out_dict['nr_pandda_pending'] = out_dict[
'nr_initial_maps_available'] - out_dict['nr_pandda_hits'] - out_dict[
'nr_pandda_reject'] - out_dict['nr_pandda_processed']
out_dict['nr_cif_files'] = len(
db.execute_statement(
"select RefinementCIF from mainTable where RefinementCIF is not (Null or '');"
))
out_dict['nr_analysis-pending'] = len(
db.execute_statement(
"select RefinementOutcome from mainTable where RefinementOutcome is '1 - Analysis Pending';"
))
out_dict['nr_pandda-models'] = len(
db.execute_statement(
"select RefinementOutcome from mainTable where RefinementOutcome is '2 - PANDDA model';"
))
out_dict['nr_in-refinement'] = len(
db.execute_statement(
"select RefinementOutcome from mainTable where RefinementOutcome is '3 - In Refinement';"
))
out_dict['nr_comp-chem-ready'] = len(
db.execute_statement(
"select RefinementOutcome from mainTable where RefinementOutcome is '4 - ComChem ready';"
))
out_dict['nr_deposition-ready'] = len(
db.execute_statement(
"select RefinementOutcome from mainTable where RefinementOutcome is '5 - Deposition ready';"
))
return out_dict
def update_data_source(db_file,xtal,db_column,status):
db=XChemDB.data_source(db_file)
db_dict={}
db_dict[db_column] = status
db.update_data_source(xtal,db_dict)
def change_links_to_selected_data_collection_outcome(
sample, data_collection_dict, data_collection_column_three_dict,
dataset_outcome_dict, initial_model_directory, data_source_file,
xce_logfile):
Logfile = XChemLog.updateLog(xce_logfile)
# find out which row was selected in respective data collection table
selected_processing_result = 'n/a'
indexes = data_collection_column_three_dict[sample][0].selectionModel(
).selectedRows()
if indexes != []: # i.e. logfile exists
for index in sorted(indexes):
selected_processing_result = index.row()
for n, entry in enumerate(data_collection_dict[sample]):
if entry[0] == 'logfile':
if entry[7] == selected_processing_result:
visit = entry[1]
run = entry[2]
autoproc = entry[4]
db_dict = entry[6]
outcome = dataset_outcome_dict[sample]
path_to_logfile = db_dict['DataProcessingPathToLogfile']
path_to_mtzfile = db_dict['DataProcessingPathToMTZfile']
mtz_filename = db_dict['DataProcessingMTZfileName']
log_filename = db_dict['DataProcessingLOGfileName']
# relative_path_to_mtzfile='./'+path_to_mtzfile.replace(initial_model_directory,'')
relative_path_to_mtzfile = './' + path_to_mtzfile.replace(
os.path.join(initial_model_directory, sample), '')
if relative_path_to_mtzfile.startswith('.//'):
relative_path_to_mtzfile = relative_path_to_mtzfile.replace(
'.//', './')
relative_path_to_logfile = './' + path_to_logfile.replace(
os.path.join(initial_model_directory, sample), '')
if relative_path_to_logfile.startswith('.//'):
relative_path_to_logfile = relative_path_to_logfile.replace(
'.//', './')
# first check if folders and files exist
# since user might do this before data are actually copied over
if os.path.isdir(
os.path.join(initial_model_directory, sample,
'autoprocessing',
visit + '-' + run + autoproc)):
db_dict['DataProcessingAutoAssigned'] = 'False'
Logfile.insert(
'changing directory to: ' +
os.path.join(initial_model_directory, sample))
os.chdir(os.path.join(initial_model_directory, sample))
# first remove old links
os.system('/bin/rm ' + sample + '.mtz 2> /dev/null')
os.system('/bin/rm ' + sample + '.log 2> /dev/null')
# make new links
# Logfile.insert('setting symlink: '+os.path.join(path_to_logfile,log_filename)+' -> '+sample+'.log')
# os.symlink(os.path.join(path_to_logfile,log_filename),sample+'.log')
# Logfile.insert('setting symlink: '+os.path.join(path_to_mtzfile,mtz_filename)+' -> '+sample+'.mtz')
# os.symlink(os.path.join(path_to_mtzfile,mtz_filename),sample+'.mtz')
Logfile.insert(
'setting relative symlink: ' +
os.path.join(relative_path_to_logfile, log_filename) +
' -> ' + sample + '.log')
os.symlink(
os.path.join(relative_path_to_logfile, log_filename),
sample + '.log')
Logfile.insert(
'setting relative symlink: ' +
os.path.join(relative_path_to_mtzfile, mtz_filename) +
' -> ' + sample + '.mtz')
os.symlink(
os.path.join(relative_path_to_mtzfile, mtz_filename),
sample + '.mtz')
# update data source
data_source = XChemDB.data_source(data_source_file)
data_source.update_insert_data_source(sample, db_dict)
else:
Logfile.insert(
'please copy data to PROJECT DIRECTORY first!')
def settings(self, xce_object):
# set XCE version
xce_object.xce_version = 'v1.4.0'
# general settings
xce_object.allowed_unitcell_difference_percent = 12
xce_object.acceptable_low_resolution_limit_for_data = 3.5
xce_object.filename_root = '${samplename}'
xce_object.data_source_set = False
xce_object.max_queue_jobs = 100
## directory settings
# set current directory and direct log to it
xce_object.current_directory = os.getcwd()
xce_object.xce_logfile = os.path.join(xce_object.current_directory,
'xce.log')
# if in the correct place, set the various directories
if 'labxchem' in xce_object.current_directory:
if len(xce_object.current_directory.split('/')
) >= 9 and xce_object.current_directory.split(
'/'
)[6] == 'processing' and xce_object.current_directory.split(
'/')[8] == 'processing':
xce_object.labxchem_directory = '/' + os.path.join(
*xce_object.current_directory.split('/')
[1:8]) # need splat operator: *
xce_object.labxchem_directory_current = '/' + os.path.join(
*xce_object.current_directory.split('/')[1:9]
) # labxchem_directory_current is where they actually have write permission
else:
xce_object.labxchem_directory = '/' + os.path.join(
*xce_object.current_directory.split('/')
[1:6]) # need splat operator: *
xce_object.labxchem_directory_current = '/' + os.path.join(
*xce_object.current_directory.split('/')
[1:7]) # need splat operator: *
# xce_object.labxchem_directory = '/' + os.path.join(
# *xce_object.current_directory.split('/')[1:6]) # need splat operator: *
xce_object.beamline_directory = os.path.join(
xce_object.labxchem_directory, 'processing', 'beamline')
if os.path.isdir(
os.path.join(xce_object.labxchem_directory, 'processing',
'analysis', 'model_building')):
xce_object.initial_model_directory = os.path.join(
xce_object.labxchem_directory, 'processing', 'analysis',
'model_building')
else:
xce_object.initial_model_directory = os.path.join(
xce_object.labxchem_directory, 'processing', 'analysis',
'initial_model')
xce_object.reference_directory = os.path.join(
xce_object.labxchem_directory, 'processing', 'reference')
xce_object.database_directory = os.path.join(
xce_object.labxchem_directory, 'processing', 'database')
xce_object.panddas_directory = os.path.join(
xce_object.labxchem_directory, 'processing', 'analysis',
'panddas')
xce_object.datasets_summary_file = os.path.join(
xce_object.database_directory,
str(os.getcwd().split('/')[5]) + '_summary.pkl')
xce_object.data_source_file = ''
xce_object.html_export_directory = os.path.join(
xce_object.labxchem_directory, 'processing', 'html')
xce_object.group_deposit_directory = os.path.join(
xce_object.labxchem_directory, 'processing',
'group_deposition')
if os.path.isfile(
os.path.join(xce_object.labxchem_directory, 'processing',
'database', 'soakDBDataFile.sqlite')):
xce_object.data_source_file = 'soakDBDataFile.sqlite'
xce_object.database_directory = os.path.join(
xce_object.labxchem_directory, 'processing', 'database')
xce_object.data_source_set = True
xce_object.db = XChemDB.data_source(
os.path.join(xce_object.database_directory,
xce_object.data_source_file))
xce_object.db.create_missing_columns()
xce_object.ccp4_scratch_directory = os.path.join(
xce_object.labxchem_directory, 'processing', 'tmp')
directory_list = [
xce_object.beamline_directory,
os.path.join(xce_object.labxchem_directory, 'processing',
'analysis'), xce_object.initial_model_directory,
xce_object.panddas_directory, xce_object.reference_directory,
xce_object.database_directory,
xce_object.ccp4_scratch_directory,
xce_object.html_export_directory,
xce_object.group_deposit_directory
]
for directory in directory_list:
if not os.path.isdir(directory):
os.mkdir(directory)
# otherwise, use the current working directory
else:
xce_object.labxchem_directory_current = xce_object.current_directory
xce_object.beamline_directory = xce_object.current_directory
xce_object.initial_model_directory = xce_object.current_directory
xce_object.reference_directory = xce_object.current_directory
xce_object.database_directory = xce_object.current_directory
xce_object.data_source_file = ''
xce_object.ccp4_scratch_directory = os.getenv('CCP4_SCR')
xce_object.panddas_directory = xce_object.current_directory
xce_object.datasets_summary_file = ''
xce_object.group_deposit_directory = xce_object.current_directory
## deposition
xce_object.deposit_dict = {}
## internal lists and dictionaries
xce_object.data_collection_list = []
xce_object.visit_list = []
xce_object.target = ''
xce_object.dataset_outcome_combobox_dict = {}
xce_object.data_collection_dict = {}
xce_object.xtal_db_dict = {}
xce_object.pandda_analyse_input_table_dict = {}
xce_object.dewar_configuration_dict = {}
xce_object.data_collection_statistics_dict = {}
xce_object.initial_model_dimple_dict = {
} # contains toggle button if dimple should be run
xce_object.reference_file_list = []
xce_object.all_columns_in_data_source = XChemDB.data_source(os.path.join
(xce_object.database_directory,
xce_object.data_source_file)) \
.return_column_list()
xce_object.albula_button_dict = {
} # using dials.image_viewer instead of albula, but keep name for dictionary
xce_object.xtalform_dict = {}
xce_object.dataset_outcome_dict = {
} # contains the dataset outcome buttons
xce_object.data_collection_table_dict = {
} # contains the dataset table
xce_object.data_collection_image_dict = {}
xce_object.data_collection_column_three_dict = {}
xce_object.datasets_summary_dict = {}
xce_object.diffraction_data_table_dict = {}
xce_object.refinement_table_dict = {}
xce_object.main_data_collection_table_exists = False
xce_object.timer_to_check_for_new_data_collection = QtCore.QTimer()
xce_object.agamemnon = False
xce_object.target_list, xce_object.visit_list = XChemMain.get_target_and_visit_list(
xce_object.beamline_directory, False)
xce_object.diffraction_data_dict = {}
## internal switches and flags
xce_object.explorer_active = 0
xce_object.coot_running = 0
xce_object.progress_bar_start = 0
xce_object.progress_bar_step = 0
xce_object.albula = None
xce_object.albula_subframes = []
xce_object.show_diffraction_image = None
xce_object.gdaLogInstructions = [0, False]
# can be any widget to be displayed in data collection summary tab
xce_object.data_collection_details_currently_on_display = None
xce_object.dataset_outcome = [
"success", "Failed - centring failed", "Failed - no diffraction",
"Failed - processing", "Failed - loop empty",
"Failed - loop broken", "Failed - low resolution",
"Failed - no X-rays", "Failed - unknown"
]
xce_object.refinement_stage = [
'0 - All Datasets', '1 - Analysis Pending', '2 - PANDDA model',
'3 - In Refinement', '4 - CompChem ready', '5 - Deposition ready',
'6 - Deposited'
]
self.set_xce_logfile(xce_object)
## external software packages
xce_object.using_remote_qsub_submission = False
xce_object.remote_qsub_submission = "/usr/bin/ssh <dls fed ID>@nx.diamond.ac.uk 'module load global/cluster; qsub'"
xce_object.update_log = XChemLog.updateLog(xce_object.xce_logfile)
xce_object.update_log.insert('new session started')
xce_object.diffraction_data_directory = xce_object.current_directory
xce_object.diffraction_data_search_info = 'n/a'
xce_object.diffraction_data_reference_mtz = 'ignore'
xce_object.html_export_directory = os.getcwd()
xce_object.external_software = XChemUtils.external_software(
xce_object.xce_logfile).check()
xce_object.second_cif_file = None
software_list = ['acedrg', 'phenix.elbow', 'grade']
for software in software_list:
if xce_object.external_software[software]:
xce_object.restraints_program = str(software)
xce_object.update_log.insert(
'will use ' + str(software) +
' for generation of ligand coordinates and'
' restraints')
else:
xce_object.restraints_program = ''
xce_object.update_log.insert(
'No program for generation of ligand coordinates and restraints available!'
)
def __init__(self,database):
self.db=XChemDB.data_source(database)