def isocluster(self, xscalefile):
if os.path.isfile(xscalefile):
comd = "xscale_isocluster -dim 3 -dmax %s %s" % (
self.jshandle['resolution'], xscalefile)
msg = "iso-clustering based on Correlation\n"
logger.info('MSG:{}'.format(msg))
try:
run_command("sxdm", self.getOutputDirectory(),
self.jshandle['user'], comd, 'isocluster.log')
except KeyError:
comd1 = "xscale_isocluster %s > isocluster.log" % xscalefile
sub.call(comd1, shell=True)
else:
self.setFailure()
return
if os.path.isfile(self.getOutputDirectory() / "isocluster.log"):
fkey = "best number of clusters"
fh = open("isocluster.log", "r")
_all = fh.readlines()
fh.close()
for lines in _all:
if "Error" in lines:
self.setFailure()
elif fkey in lines:
line = lines.split(':')
val = line[-1].strip('\n')
self.results["clusters"] = val.strip(' ')
else:
pass
if os.path.isfile(str(self.getOutputDirectory(), "XSCALE.1.INP")):
shutil.copyfile("XSCALE.1.INP", "XSCALE.INP")
msg = "xscale-ing over 1st cluster only..\n"
logger.info('MSG:{}'.format(msg))
try:
run_command("sxdm", self.getOutputDirectory(),
self.jshandle['user'], Merging._command,
self.getLogFileName())
except KeyError:
sub.call(Merging._command, shell=True)
try:
indict = {"LPfile": "XSCALE.LP"}
xscale_parse = OutputParser(indict)
xscale_parse.parse_xscale_output(indict)
logger.info('stat_dict:{}'.format(xscale_parse.results))
shutil.copyfile("XSCALE.INP", "cluster1.INP")
shutil.copyfile("XSCALE.LP", "cluster1.LP")
shutil.copyfile("XSCALE.HKL", "cluster1.HKL")
self.results['iso-cluster'] = xscale_parse.results
except OSError:
err = "xscaling after iso-clustering may not work\n"
logger.info('OSError:{}'.format(err))
self.setFailure()
return
return
def Isa_select(self, config):
# method to perform ISa based selection of 'good' HKL files for next round of XSCALEing
isa_threshold = self.jshandle.get('isa_cutoff', 3.0)
self.results['isa_selected'] = []
if os.path.isfile("XSCALE.LP"):
fh = open("XSCALE.LP", 'r')
all_lines = fh.readlines()
fh.close()
try:
start = all_lines.index(
' a b ISa ISa0 INPUT DATA SET\n')
end = all_lines.index(
' (ASSUMING A PROTEIN WITH 50% SOLVENT)\n')
# select the table with ISa values from the file..
isa_list = all_lines[start + 1:end - 3]
# except (ValueError, IndexError) as err:
# error = "check if XSCALE ran properly or XSCALE.INP screwed up \n"
# logger.error(error)
for lines in isa_list:
line = lines.split()
try:
if float(line[2]) > float(isa_threshold):
self.results['isa_selected'].append(line[4])
except IndexError:
pass
self.results['isa_selected'] = Merging.sort_xtal(
self.results['isa_selected'])
self.create_inp(self.results['isa_selected'], config)
msg = "%d xtals selected by ISa-cut out of %d xtals\n" % (
len(self.results['isa_selected']),
len(self.results['hklpaths_found']))
self.results['xtals_after_isa'] = len(
self.results['isa_selected'])
logger.info('MSG:{}'.format(msg))
try:
# user = self.jshandle.get('user', " ")
run_command("sxdm", self.getOutputDirectory(),
self.jshandle['user'], Merging._command,
self.getLogFileName())
except KeyError as e:
sub.call(Merging._command, shell=True)
except Exception as e:
logger.error(e)
self.setFailure()
else:
err = "XSCALE.LP file does not exist"
logger.error(err)
self.setFailure()
return
def xdsconv(self, inData):
fname = inData['xds_ascii']
linkname = "data.hkl"
res = inData.get('resolution', 0.0)
form = inData['mtz_format']
if os.path.isfile(fname):
try:
os.symlink(fname, linkname)
except OSError as e:
if e.errno == errno.EEXIST:
os.remove(linkname)
os.symlink(fname, linkname)
else:
logger.info('xdsconv-error:{}'.format(e))
else:
logger.info('xdsconv_error:{}'.format('file does not exist'))
namelist = os.path.basename(fname).split('.')
rootname = namelist[0]
mtzname = str(rootname + "_" + form + ".mtz")
xdsconv_cmd = 'xdsconv > /dev/null'
f2mtz_cmd = 'f2mtz HKLOUT %s < F2MTZ.INP > /dev/null' % mtzname
fh = open('XDSCONV.INP', 'w')
fh.write("OUTPUT_FILE=temp.hkl %s\n" % form)
fh.write("INPUT_FILE=%s\n" % linkname)
fh.write("FRIEDEL'S_LAW=%s\n" % self.results['anom'])
fh.write("INCLUDE_RESOLUTION_RANGE=50 %f\n" % res)
fh.write("GENERATE_FRACTION_OF_TEST_REFLECTIONS= 0.05\n")
fh.close()
logger.info("running xdsconv")
try:
run_command("sxdm", self.getOutputDirectory(), inData['user'],
xdsconv_cmd, 'xdsconv.log')
except KeyError:
sub.call(xdsconv_cmd, shell=True)
time.sleep(2)
if os.path.isfile('F2MTZ.INP'):
try:
run_command("sxdm", self.getOutputDirectory(), inData['user'],
f2mtz_cmd, 'xdsconv.log')
except KeyError:
sub.call(f2mtz_cmd, shell=True)
time.sleep(2)
os.remove('temp.hkl')
else:
logger.info('f2mtz_error:{}'.format('xdsconv did not run\n'))
return
def main():
parser = ArgumentParser('Launch test query suit')
parser.add_argument('TEST_SUIT_PATH', help='test suite in .csv')
parser.add_argument('--algo',
help='algorithms for running',
nargs='+',
required=True)
parser.add_argument('--prof', help='memory profiler', default=None)
parser.add_argument('--pid', help='redis pid', type=int, default=None)
parser.add_argument('--host', help='redis host name', default='localhost')
parser.add_argument('--port', help='redis port', default=6379)
parser.add_argument('--out',
help='response output dir path',
default='cfpq_results')
parser.add_argument('--repeat', help='repeat count', type=int, default=5)
args = parser.parse_args()
mem_profiler_factory = None
if args.prof == "gpu":
mem_profiler_factory = GpuMemDeltaProfiler
redis_instance = redis.Redis(args.host, args.port)
suite = load_suite(args.TEST_SUIT_PATH)
statistic = defaultdict(list)
try:
suite_range = tqdm(suite)
for graph_name, grammar_path in suite_range:
for algo in args.algo:
suite_range.set_description(
f'Processing {graph_name} {Path(grammar_path).name} {algo}'
)
res_repeats = list()
for _ in tqdm(range(args.repeat), leave=False):
res = run_command(
redis_instance,
('graph.cfg', algo, graph_name, grammar_path),
args.pid, mem_profiler_factory)
res_repeats.append(parse_result(res))
statistic['graph_name'].append(graph_name)
statistic['grammar'].append(Path(grammar_path).name)
statistic['algo'].append(algo)
for key, value in create_statistic(res_repeats).items():
statistic[key].append(value)
except (KeyboardInterrupt, redis.exceptions.ConnectionError):
print("Interrupted, saving results...")
except redis.exceptions.ConnectionError:
print("Connection error, saving results...")
path_result = f'{args.out}/{Path(args.TEST_SUIT_PATH).stem}'
if not os.path.isdir(path_result):
os.makedirs(path_result)
df = pandas.DataFrame(statistic)
df.columns = map(snakecase, df.columns)
df.to_csv(
f"{path_result}/{datetime.datetime.now().strftime('%Y-%m-%d_%H:%M')}_stat.csv"
)
def pointless(fname, refname, dirname=None, user=None):
success = False
input_cmd = """pointless XDSIN %s -HKLREF %s << EOF\nTESTFIRSTFILE\nsetting symmtery-based\nEOF""" % (
fname, refname)
try:
fh = open("index_check", 'w')
fh.write("#!/bin/bash\n\n")
fh.write(input_cmd)
fh.close()
chmod_cmd = "chmod +x index_check"
ind_cmd = "./index_check > pointless.log"
try:
run_command('sxdm', dirname, user, chmod_cmd, 'merge.log')
run_command('sxdm', dirname, user, ind_cmd, 'merge.log')
except (OSError, TypeError, Exception) as e:
sub.call(chmod_cmd, shell=True)
sub.call(ind_cmd, shell=True)
success = True
except Exception as err:
logger.error(err)
success = False
return success
def aniso_check(self):
point_cmd = "pointless -xdsin noSelect.HKL hklout noSelect_point.mtz"
try:
run_command("sxdm", self.getOutputDirectory(),
self.jshandle['user'], point_cmd, 'pointless.log')
except KeyError:
point_cmd1 = "pointless -xdsin noSelect.HKL hklout noSelect_point.mtz > pointless.log"
sub.call(point_cmd1, shell=True)
time.sleep(2)
if os.path.isfile(str(self.getOutputDirectory() /
"noSelect_point.mtz")):
fh = open("onlymerge.inp", 'w')
fh.write("ONLYMERGE\n")
try:
from iotbx import reflection_file_reader
mtzfile = reflection_file_reader.any_reflection_file(
"noSelect_point.mtz")
mtz_content = mtzfile.file_content()
col_labels = mtz_content.column_labels()
if "BATCH" in col_labels:
n_batches = mtz_content.n_batches()
fh.write("run 1 batch %d to %d\n" % (1, n_batches))
else:
pass
except (ImportError, RuntimeError) as err:
logger.info('iotbx-import-error:{}'.format(err))
fh.write("run 1 batch %d to %d\n" % (1, 50))
fh.close()
aim_cmd = "aimless hklin noSelect_point.mtz hklout noSelect_aim.mtz < onlymerge.inp"
try:
run_command("sxdm", self.getOutputDirectory(),
self.jshandle['user'], aim_cmd, 'aimless.log')
except (OSError, TypeError, Exception) as e:
aim_cmd1 = aim_cmd + '| tee aimless.log'
sub.call(aim_cmd1, shell=True)
else:
err = "Could be pointless did not run\n"
logger.info('aimless-error:{}'.format(err))
self.setFailure()
return
try:
fh = open("aimless.log", 'r')
_all = fh.readlines()
fh.close()
except (OSError, IOError):
err = "aimless.log file doesn't exist"
logger.info('OSError:{}'.format(err))
self.setFailure()
return
keyword = "Estimated maximum resolution limits"
for lines in _all:
if "Error" in lines:
self.setFailure()
elif keyword in lines:
line = lines.split(',')
try:
a = line[1].split(':')[1]
b = line[2].split(':')[1]
c = line[3].split(':')[1].strip('\n')
aniso_string = "a* =%s, b*=%s, c*=%s" % (a, b, c)
self.results['anisotropicity'] = aniso_string
logger.info('Anisotropy:{}'.format(aniso_string))
except IndexError:
hk_plane = line[1].split(':')[1]
l_axis = line[2].split(':')[1].strip('\n')
aniso_string = "hk-plane = %s, l-axis = %s" % (hk_plane,
l_axis)
self.results['anisotropicity'] = aniso_string
else:
pass
return
def xscale_for_sx(self):
self.find_HKLs()
config = dict()
try:
self.outdir_make()
except OSError:
err = "adm folder either not there or not accessible\n"
logger.error(err)
self.setFailure()
return
try:
if len(self.results['hklpaths_found']) < 1000:
self.indexing_()
else:
msg = "Too many hkls, so skipping indexing check"
logger.info('MSG:{}'.format(msg))
pass
indict = {
"listofHKLfiles": self.results['hklpaths_found'],
"fom": 'rmeas'
}
sc = ScaleUtils(indict)
sc.ref_choice(indict)
logging.info('lowest-Rmeas file: %s' % sc.results['reference'])
indata_ascii = {"xds_ascii": sc.results['reference']}
ref_for_cell_sg = ASCII(indata_ascii)
ref_for_cell_sg.get_data(indata_ascii)
self.results['space_group'] = space_group[
ref_for_cell_sg.results['spg']][0]
self.results['unit-cell'] = ref_for_cell_sg.results['unit_cell']
self.results['friedel'] = ref_for_cell_sg.results['anom']
config['reference'] = sc.results['reference']
self.create_file_links()
self.create_inp(self.results['filelinks'], config)
msg = "Running 1st round of xscale-ing with Rmeas based ranking\n"
logger.info('MSG:{}'.format(msg))
try:
run_command("sxdm", self.getOutputDirectory(),
self.jshandle['user'], Merging._command,
self.getOutputDirectory())
except KeyError:
sub.call(Merging._command, shell=True)
try:
indict = {"LPfile": "XSCALE.LP"}
xscale_parse = OutputParser(indict)
xscale_parse.parse_xscale_output(indict)
logger.info('stat_dict:{}'.format(xscale_parse.results))
self.results['no_selection'] = xscale_parse.results
shutil.copyfile("XSCALE.INP", "noSelect.INP")
shutil.copyfile("XSCALE.LP", "noSelect.LP")
shutil.copyfile("XSCALE.HKL", "noSelect.HKL")
except OSError:
err = "xscaling may not have run\n"
logger.error(err)
self.setFailure()
return
msg = "running xscale after ISa selection\n"
logger.info(msg)
self.Isa_select(config)
try:
indict = {"LPfile": "XSCALE.LP"}
xscale_parse = OutputParser(indict)
xscale_parse.parse_xscale_output(indict)
logger.info('stat_dict:{}'.format(xscale_parse.results))
self.results['ISa_selection'] = xscale_parse.results
shutil.copyfile("XSCALE.INP", "ISa_Select.INP")
shutil.copyfile("XSCALE.LP", "ISa_Select.LP")
shutil.copyfile("XSCALE.HKL", "ISa_Select.HKL")
except OSError:
err = "xscaling after ISa selection may not work\n"
logger.info('OSError:{}'.format(err))
self.setFailure()
return
celldict = {"listofHKLfiles": self.results['filelinks']}
celler = Cell(celldict)
celler.setter(celldict)
if len(celler.results['hklList']) > 200:
celler.cell_analysis(celldict)
self.results['cell_array'] = celler.results['cell_ar'].tolist()
self.results['histogram'] = celler.dict_for_histogram()
else:
celler.clustering(celldict)
self.results['cell_array'] = celler.results[
'cell_ar_best_cluster'].tolist()
self.results['dendro_labels'] = celler.results['data_points']
mode_cells = str(celler.results['a_mode']) + " " + str(celler.results['b_mode']) + ' ' + str(celler.results['c_mode']) + \
" " + str(celler.results['al_mode']) + " " + str(celler.results['be_mode']) + " " + str(celler.results['ga_mode'])
# self.results['unit-cell'] = mode_cells
# convert dendro dictionary for easy plottable dictionary in adp tracker
try:
self.results['cell_dendrogram'] = dendro2highcharts(
celler.results['dendo'])
# self.scale_results['hclust_matrix'] = celler.hclust_matrix
self.results['cell_n_clusters'] = celler.results[
'n_clusters_cell']
except Exception as e:
logger.info('skipping-dendro-cell:{}'.format(e))
try:
config['unit_cell'] = mode_cells
config['space_group'] = ref_for_cell_sg.results['spg']
self.create_inp(celler.results['cell_select'], config)
msg = "xscaling after cell-analysis and rejecting outliers\n"
msg += "%d xtals got selected after cell analysis out of %d xtals" \
%(len(celler.results['cell_select']), len(self.results['hklpaths_found']))
logger.info('MSG:{}'.format(msg))
try:
run_command("sxdm", self.getOutputDirectory(),
self.jshandle['user'], Merging._command,
self.getLogFileName())
except (OSError, TypeError, KeyError) as e:
sub.call(Merging._command, shell=True)
self.results['xtals_after_cell'] = len(
celler.results['cell_select'])
except Exception as e:
logger.info('Error:{}'.format(e))
self.setFailure()
try:
indict = {"LPfile": "XSCALE.LP"}
xscale_parse = OutputParser(indict)
xscale_parse.parse_xscale_output(indict)
logger.info('stat_dict:{}'.format(xscale_parse.results))
self.results['cell_selection'] = xscale_parse.results
shutil.copyfile("XSCALE.INP", "Cell_Select.INP")
shutil.copyfile("XSCALE.LP", "Cell_Select.LP")
shutil.copyfile("XSCALE.HKL", "Cell_Select.HKL")
except OSError:
err = "xscaling after Cell selection may not work\n"
logger.info('OSError:{}'.format(err))
ccDict = {"xds_ascii": 'noSelect.HKL'}
CC = corc.CCestimator(ccDict)
logger.info('ASCII loaded')
CC.cc_select(fom='pcc')
try:
self.results['cc_dendrogram'] = dendro2highcharts(
CC.results['cc_dendo'])
self.results['cc_n_clusters'] = CC.results['n_clusters_cc']
self.results['hkl_cc_sorted'] = CC.results['cc_cluster_list']
self.results['xtals_after_pCC'] = len(
self.results['hkl_cc_sorted'])
msg = "pair-correlation sorting over Isa_select: %d hkls" % self.results[
'xtals_after_pCC']
logger.info('MSG:{}'.format(msg))
self.create_inp(self.results['hkl_cc_sorted'], config)
except Exception as err:
logger.info('cc-dendro-empty:{}'.format(err))
try:
run_command("sxdm", self.getOutputDirectory(),
self.jshandle['user'], Merging._command,
self.getLogFileName())
except (OSError, TypeError, Exception) as e:
sub.call(Merging._command, shell=True)
try:
indict = {"LPfile": "XSCALE.LP"}
xscale_parse = OutputParser(indict)
xscale_parse.parse_xscale_output(indict)
logger.info('stat_dict:{}'.format(xscale_parse.results))
self.results['pCC_selection'] = xscale_parse.results
shutil.copyfile("XSCALE.INP", "pCC_Select.INP")
shutil.copyfile("XSCALE.LP", "pCC_Select.LP")
shutil.copyfile("XSCALE.HKL", "pCC_Select.HKL")
except OSError:
err = "xscaling after pair-correlation selection may not work\n"
logger.info('OSError:{}'.format(err))
self.setFailure()
except Exception as e:
logger.info('Error: {}'.format(e))
self.setFailure()
return
def xscale_for_sad(self):
self.find_HKLs()
self.outdir_make()
config = dict()
if len(self.results['hklpaths_found']) > 0:
if len(self.results['hklpaths_found']) < 1000:
self.indexing_()
else:
logger.info('Too many hkls, so skipping the indexing check')
pass
self.create_file_links()
indict = {
"listofHKLfiles": self.results['hklpaths_found'],
"fom": 'bfac'
}
sc = ScaleUtils(indict)
sc.ref_choice(indict)
logging.info('lowest-Bfactor file: %s' % sc.results['reference'])
indata_ascii = {"xds_ascii": sc.results['reference']}
ref_for_cell_sg = ASCII(indata_ascii)
ref_for_cell_sg.get_data(indata_ascii)
self.results['space_group'] = space_group[
ref_for_cell_sg.results['spg']][0]
self.results['unit-cell'] = ref_for_cell_sg.results['unit_cell']
self.results['friedel'] = ref_for_cell_sg.results['anom']
config['reference'] = sc.results['reference']
self.create_file_links()
self.create_inp(self.results['filelinks'], config)
else:
# self.create_inp(self.results['hklpaths_found'], config)
logger.error("No hkl file found")
msg = "xscale-ing of native SAD data\n"
logger.info('MSG:{}'.format(msg))
try:
run_command("sxdm", self.getOutputDirectory(),
self.jshandle['user'], Merging._command,
self.getLogFileName())
except KeyError:
sub.call(Merging._command, shell=True)
try:
indict = {"LPfile": "XSCALE.LP"}
xscale_parse = OutputParser(indict)
xscale_parse.parse_xscale_output(indict)
logger.info('stat_dict:{}'.format(xscale_parse.results))
self.results['nSAD_xscale_stats'] = xscale_parse.results
shutil.copyfile('XSCALE.HKL', 'nSAD.HKL')
shutil.copyfile('XSCALE.LP', 'nSAD.LP')
except Exception as err:
logger.error(err)
self.setFailure()
self.Isa_select(config)
shutil.copyfile('XSCALE.LP', 'ISa_Select.LP')
shutil.copyfile('XSCALE.HKL', 'ISa_Select.HKL')
indict = {"LPfile": "ISa_Select.LP"}
xscale_parse = OutputParser(indict)
xscale_parse.parse_xscale_output(indict)
self.results['ISa_selection'] = xscale_parse.results
return