def crystfel_viewindex_pick(self):
file = cfel_file.dialog_pickfile(path='../indexing/streams',
filter='*.stream',
qtmainwin=self)
if file != '':
cmdarr = ['cxiview', '-s', file]
cfel_file.spawn_subprocess(cmdarr)
def show_peakogram(self):
runs = self.selected_runs()
if len(runs['run']) == 0:
return
pkfile = runs['path'][0] + 'peaks.txt'
cmdarr = ['peakogram.py', '-i', pkfile]
cfel_file.spawn_subprocess(cmdarr)
def start_crawler(self):
cmdarr = [
'cheetah-crawler.py', '-l', self.compute_location['location'],
'-d', self.config['xtcdir'], '-c', self.config['hdf5dir'], '-i',
'../indexing/'
]
cfel_file.spawn_subprocess(cmdarr) #, shell=True)
def crystfel_cellexplore_pick(self):
file = cfel_file.dialog_pickfile(path='../indexing/streams',
filter='*.stream',
qtmainwin=self)
if file != '':
cmdarr = ['cell_explorer', file]
cfel_file.spawn_subprocess(cmdarr)
def copy_darkcal(self):
runs = self.selected_runs()
if len(runs['run']) == 0:
return
for path in runs['path']:
path += '*detector0-darkcal.h5'
for dkcal in glob.iglob(path):
cmdarr = ['cp', dkcal, '../calib/darkcal/.']
cfel_file.spawn_subprocess(cmdarr)
def set_current_badpix(self):
file = cfel_file.dialog_pickfile(path='../calib/mask',
filter='*.h5',
qtmainwin=self)
if file != '':
file = os.path.relpath(file)
basename = os.path.basename(file)
dirname = os.path.dirname(file)
cmdarr = ['ln', '-fs', basename, '../calib/mask/current-badpix.h5']
cfel_file.spawn_subprocess(cmdarr)
def crystfel_cellexplore(self):
runs = self.selected_runs()
if len(runs['run']) == 0:
return
dir = '../indexing/' + runs['directory'][0]
stream = glob.glob(dir + '/*.stream')
if len(stream) is 0:
return
cmdarr = ['cell_explorer', stream[0]]
cfel_file.spawn_subprocess(cmdarr)
def set_cxiview_geom(self):
file = cfel_file.dialog_pickfile(path='../calib/geometry',
filter='*.geom',
qtmainwin=self)
if file != '':
file = os.path.relpath(file)
basename = os.path.basename(file)
dirname = os.path.dirname(file)
cmdarr = [
'ln', '-fs', basename,
'../calib/geometry/current-geometry.geom'
]
cfel_file.spawn_subprocess(cmdarr)
def run_XFEL_detectorcalibration(self):
runs = self.selected_runs()
if len(runs) is 0:
print('No runs selected')
return
for i, run in enumerate(runs['run']):
print(
'------------ Start XFEL detector calibration script ------------'
)
#cmdarr = [self.config['process'], run]
cmdarr = ["../process/calibrate_euxfel.sh", run]
cfel_file.spawn_subprocess(cmdarr, shell=True)
def show_selected_images(self, filepat, field='data/data'):
runs = self.selected_runs()
# No run selected
if len(runs['run']) == 0:
return
file = runs['path'][0] + filepat
# Display if some file matches pattern (avoids error when no match)
if len(glob.glob(file)) != 0:
cmdarr = [
'cxiview.py', '-g', self.config['geometry'], '-e', field, '-i',
file
]
cfel_file.spawn_subprocess(cmdarr)
else:
print("File does not seem to exist:")
print(file)
def maskmaker(self):
print("Mask maker selected")
runs = self.selected_runs()
if len(runs['run']) == 0:
return
file = runs['path'][0]
file += '*detector0-class0-sum.h5'
field = 'data/data'
if len(glob.glob(file)) != 0:
file = glob.glob(file)[0]
cmdarr = [
'maskMakerGUI.py', '-g', self.config['geometry'], file, field
]
cfel_file.spawn_subprocess(cmdarr)
else:
print("File does not seem to exist:")
print(file)
def set_darkrun(self):
# This bit repeats copy_darkcal, but allows only one run and remembers filename for symlink
runs = self.selected_runs()
if len(runs['run']) == 0:
return
if len(runs['run']) > 1:
print('Can only select one run')
return
# Copy files
path = runs['path'][0]
path += '*detector0-darkcal.h5'
for dkcal in glob.iglob(path):
cmdarr = ['cp', dkcal, '../calib/darkcal/.']
cfel_file.spawn_subprocess(cmdarr)
# Create symbolic link
file = os.path.relpath(dkcal)
basename = os.path.basename(file)
cmdarr = ['ln', '-fs', basename, '../calib/darkcal/current-darkcal.h5']
cfel_file.spawn_subprocess(cmdarr)
def modify_EuXFEL_cheetah_config_files(self):
# Dialog box
result, ok = gui_dialogs.configure_cheetah_euxfel_gui.configure_cheetah_dialog(
)
if ok == False:
print('Configuration scripts remain unchanged')
return
# This is needed because on startup we are one directory up from cheetah/gui, but when running we are in cheetah/gui
dir = os.getcwd()
if "cheetah/gui" in dir:
prefix = '..'
else:
prefix = 'cheetah'
print(result)
detectorType = result["detectorType"]
detectorName = result["detectorName"]
# Modify detector type and name in .ini files
print("Modifying detector type and name...")
for file in glob.iglob(prefix + '/process/*.ini'):
cmd = [
"sed", "-i", "-r",
"/^detectorType/s/(detectorType=).*/\\1" + detectorType + "/", file
]
cfel_file.spawn_subprocess(cmd, wait=True)
cmd = [
"sed", "-i", "-r",
"/^detectorName/s/(detectorName=).*/\\1" + detectorName + "/", file
]
cfel_file.spawn_subprocess(cmd, wait=True)
# Modify detector name in AGIPD preprocess-dk script
file = prefix + '/calib/agipd/scripts/preprocess-dk'
print('Modifying ', file)
cmd = [
"sed", "-i", "-r", "s/(--detector_string ).*/\\1" + detectorName + "/",
file
]
cfel_file.spawn_subprocess(cmd, wait=True)
return
def extract_euxfel_template(self):
# Deduce experiment number, etc using de-referenced paths
realdir = os.getcwd()
print('os.getcwd() = ', realdir)
# Now for some EuXFEL-specific directory stuff
# /gpfs/exfel/u/scratch/SPB/201701/p002017/anton/cheetah-test
# to
# /gpfs/exfel/exp/SPB/201701/p002017
ss = realdir.split('/')
ss = ss[1:]
print(ss)
instr = ss[4]
run = ss[5]
expt = ss[6]
tempdir = str.join('/', ss[7:])
ssnew = ss
ssnew[2] = 'exp'
ssnew[3:6] = ss[4:7]
absroot = '/' + str.join('/', ssnew[0:6])
absdir = absroot + '/scratch/' + tempdir
datadir = absroot + '/raw'
userdir = absdir + '/cheetah'
print('Deduced experiment information:')
print(' Sensed directory: ', realdir)
print(' Preferred path: ', absdir)
print(' Instrument: ', instr)
print(' Experiment: ', expt)
print(' Proposal round: ', run)
print(' Data directory: ', datadir)
print(' Output directory: ', userdir)
#
# QMessageBox for confirmation before proceeding
#
msgBox = PyQt5.QtWidgets.QMessageBox()
str1 = []
str1.append('<b>Instrument:</b> ' + str(instr))
str1.append('<b>Experiment:</b> ' + str(expt))
str1.append('<b>Data directory:</b> ' + str(datadir))
str1.append('<b>Output directory:</b> ' + str(userdir))
# str1.append('<b>Relative path:</b> '+ str(dir))
str2 = str.join('<br>', str1)
msgBox.setText(str2)
msgBox.setInformativeText('<b>Proceed?</b>')
msgBox.addButton(PyQt5.QtWidgets.QMessageBox.Yes)
msgBox.addButton(PyQt5.QtWidgets.QMessageBox.Cancel)
msgBox.setDefaultButton(PyQt5.QtWidgets.QMessageBox.Yes)
ret = msgBox.exec_()
if ret == PyQt5.QtWidgets.QMessageBox.Cancel:
print("So long and thanks for all the fish.")
self.exit_gui()
#
# Extract the template
#
# Unpack template
print('>---------------------<')
print('Extracting template...')
template = '/gpfs/cfel/cxi/common/public/development/cheetah/template.tar'
cmd = ['tar', '-xf', template]
cfel_file.spawn_subprocess(cmd, wait=True)
print("Done")
# Fix permissions
print('>---------------------<')
print('Fixing permissions...')
cmd = ['chmod', '-R', 'ug+w', 'cheetah/']
cfel_file.spawn_subprocess(cmd, wait=True)
cmd = ['chmod', 'ug+x', 'cheetah/process/process', 'cheetah/process/hitfinder']
cfel_file.spawn_subprocess(cmd, wait=True)
print("Done")
#
# Modify configuration files
#
# Modify gui/crawler.config
print('>---------------------<')
file = 'cheetah/gui/crawler.config'
print('Modifying ', file)
# Replace XTC directory with the correct location using sed
xtcsedstr = str.replace(datadir, '/', '\\/')
cmd = ["sed", "-i", "-r", "s/(xtcdir=).*/\\1" + xtcsedstr + "/", file]
cfel_file.spawn_subprocess(cmd, wait=True)
print('>-------------------------<')
cmd = ['cat', file]
cfel_file.spawn_subprocess(cmd, wait=True)
print('>-------------------------<')
# Modify process/process
file = 'cheetah/process/process'
print('Modifying ', file)
exptstr = instr+'/'+run+'/'+expt
expsedstr = str.replace(exptstr, '/', '\\/')
cmd = ["sed", "-i", "-r", "s/(expt=).*/\\1" + expsedstr + "/", file]
cfel_file.spawn_subprocess(cmd, wait=True)
print('>-------------------------<')
cmd = ['head', file]
cfel_file.spawn_subprocess(cmd, wait=True)
print('>-------------------------<')
print("Working directory: ")
print(os.getcwd() + '/cheetah')
return
def index_runs(guiself, dirs=None, nocell=False, geopt=False):
# Just some info
print("Will process the following directories:")
print(dirs)
# Select geometry file and remember it
geomfile = guiself.lastgeom
if geomfile is None:
geomfile = cfel_file.dialog_pickfile(path='../calib/geometry',
filter='*.geom',
qtmainwin=guiself)
if geomfile is '':
return
guiself.lastgeom = geomfile
# Default unit cell file
cell = guiself.lastcell
# This bit handles which recipe is selected
# (including nocell and geopt options)
recipe = guiself.lastindex
if nocell:
recipe = '../process/index_nocell.sh'
if geopt:
recipe = '../process/index_geopt.sh'
# Launch dialog box for CrystFEL options
dialog_in = {
'pdb_files':
glob.glob('../calib/pdb/*.pdb') + glob.glob('../calib/pdb/*.cell'),
'geom_files':
glob.glob('../calib/geometry/*.geom'),
'recipe_files':
glob.glob('../process/index*.sh'),
'default_geom':
geomfile,
'default_cell':
cell,
'default_recipe':
recipe
}
if dialog_in['default_cell'] is None or not cell in dialog_in['pdb_files']:
dialog_in['default_cell'] = dialog_in['pdb_files'][0]
dialog_out, ok = gui_dialogs.run_crystfel_dialog.dialog_box(dialog_in)
# Exit if cancel was pressed
if ok == False:
return
# Remember selections for later
pdbfile = dialog_out['pdbfile']
geomfile = dialog_out['geomfile']
recipefile = dialog_out['recipefile']
guiself.lastcell = pdbfile
guiself.lastgeom = geomfile
if not nocell and not geopt:
guiself.lastindex = recipefile
#geomfile = os.path.abspath(geomfile)
#
# Loop through selected directories
# Much of this repeats what is in index-nolatt.... simplify later
#
for dirbase in dirs:
# Data location and destination location
print(dir)
h5dir = "../hdf5/" + dirbase
indexdir = "../indexing/" + dirbase
# Remove contents of any existing directory then recreate directory
shutil.rmtree(indexdir, ignore_errors=True)
os.makedirs(indexdir, exist_ok=True)
# Use find to create file list
cmd = 'find ' + os.path.abspath(
h5dir) + ' -name \*.cxi > ' + indexdir + '/files.lst'
print(cmd)
os.system(cmd)
# Copy scripts and calibrations to target directory
cmdarr = ['cp', recipefile, indexdir + '/.']
cfel_file.spawn_subprocess(cmdarr, wait=True)
cmdarr = ['cp', geomfile, indexdir + '/.']
cfel_file.spawn_subprocess(cmdarr, wait=True)
cmdarr = ['cp', pdbfile, indexdir + '/.']
cfel_file.spawn_subprocess(cmdarr, wait=True)
# Send indexing command to batch farm
qlabel = 'indx-' + dirbase[1:5]
logfile = 'bsub.log'
abspath = os.path.abspath(indexdir) + '/'
#Version 0
#bsub_cmd = ['bsub', '-q', 'psanaq', '-x', '-J', qlabel, '-o', logfile, '-cwd', abspath, 'source', './'+os.path.basename(recipefile), dirbase, os.path.basename(pdbfile), os.path.basename(geomfile)]
#Version 1
bsub_cmd = [
'../process/queue.sh', qlabel, abspath, 'source',
'./' + os.path.basename(recipefile), dirbase,
os.path.basename(pdbfile),
os.path.basename(geomfile)
]
# Submit it
cfel_file.spawn_subprocess(bsub_cmd)
print(">------------------------------<")
return
def index_runs(guiself, dirs=None, nocell=False, geopt=False):
# Just some info
print("Will process the following directories:")
print(dirs)
# Select geometry file and remember it
geomfile = guiself.lastgeom
if geomfile is None:
geomfile = cfel_file.dialog_pickfile(path='../calib/geometry', filter='*.geom', qtmainwin=guiself)
if geomfile is '':
return
guiself.lastgeom = geomfile
# Default unit cell file
cell=guiself.lastcell
# This bit handles which recipe is selected
# (including nocell and geopt options)
recipe = guiself.lastindex
if nocell:
recipe = '../process/index_nocell.sh'
if geopt:
recipe = '../process/index_geopt.sh'
# Launch dialog box for CrystFEL options
dialog_in = {
'pdb_files' : glob.glob('../calib/pdb/*.pdb')+glob.glob('../calib/pdb/*.cell'),
'geom_files' : glob.glob('../calib/geometry/*.geom'),
'recipe_files' : glob.glob('../process/index*.sh'),
'default_geom' : geomfile,
'default_cell' : cell,
'default_recipe' : recipe
}
if dialog_in['default_cell'] is None or not cell in dialog_in['pdb_files']:
dialog_in['default_cell'] = dialog_in['pdb_files'][0]
dialog_out, ok = gui_dialogs.run_crystfel_dialog.dialog_box(dialog_in)
# Exit if cancel was pressed
if ok == False:
return
# Remember selections for later
pdbfile = dialog_out['pdbfile']
geomfile = dialog_out['geomfile']
recipefile = dialog_out['recipefile']
guiself.lastcell = pdbfile
guiself.lastgeom = geomfile
if not nocell and not geopt:
guiself.lastindex = recipefile
#geomfile = os.path.abspath(geomfile)
#
# Loop through selected directories
# Much of this repeats what is in index-nolatt.... simplify later
#
for dirbase in dirs:
# Data location and destination location
print(dir)
h5dir = "../hdf5/" + dirbase
indexdir = "../indexing/" + dirbase
# Remove contents of any existing directory then recreate directory
shutil.rmtree(indexdir, ignore_errors=True)
os.makedirs(indexdir, exist_ok=True)
# Use find to create file list
cmd = 'find ' + os.path.abspath(h5dir) + ' -name \*.cxi > ' + indexdir + '/files.lst'
print(cmd)
os.system(cmd)
# Copy scripts and calibrations to target directory
cmdarr = ['cp', recipefile , indexdir + '/.']
cfel_file.spawn_subprocess(cmdarr, wait=True)
cmdarr = ['cp', geomfile, indexdir + '/.']
cfel_file.spawn_subprocess(cmdarr, wait=True)
cmdarr = ['cp', pdbfile, indexdir + '/.']
cfel_file.spawn_subprocess(cmdarr, wait=True)
# Send indexing command to batch farm
qlabel = 'indx-'+dirbase[1:5]
logfile = 'bsub.log'
abspath = os.path.abspath(indexdir)+'/'
#Version 0
#bsub_cmd = ['bsub', '-q', 'psanaq', '-x', '-J', qlabel, '-o', logfile, '-cwd', abspath, 'source', './'+os.path.basename(recipefile), dirbase, os.path.basename(pdbfile), os.path.basename(geomfile)]
#Version 1
bsub_cmd = ['../process/queue.sh', qlabel, abspath, 'source', './'+os.path.basename(recipefile), dirbase, os.path.basename(pdbfile), os.path.basename(geomfile)]
# Submit it
cfel_file.spawn_subprocess(bsub_cmd)
print(">------------------------------<")
return
def extract_lcls_template(self):
# Deduce experiment number, etc using de-referenced paths
# Assumes the file path follows the pattern: /reg/d/psdm/cxi/cxij4915/scratch/...
realdir = os.getcwd()
# Now for some LCLS-specific stuff
ss = realdir.split('/')
ss = ss[1:]
ss[1] = 'd'
ss[2] = 'psdm'
instr = ss[3]
expt = ss[4]
xtcdir = '/' + str.join('/', ss[0:5]) + '/xtc'
userdir = '/' + str.join('/', ss) + '/cheetah'
print('Deduced experiment information:')
print(' Relative path: ', dir)
print(' Absolute path: ', realdir)
print(' Instrument: ', instr)
print(' Experiment: ', expt)
print(' XTC directory: ', xtcdir)
print(' Output directory: ', userdir)
#
# QMessageBox for confirmation before proceeding
#
msgBox = PyQt5.QtWidgets.QMessageBox()
str1 = []
str1.append('<b>Instrument:</b> ' + str(instr))
str1.append('<b>Experiment:</b> ' + str(expt))
str1.append('<b>XTC directory:</b> ' + str(xtcdir))
str1.append('<b>Output directory:</b> ' + str(userdir))
# str1.append('<b>Relative path:</b> '+ str(dir))
str2 = str.join('<br>', str1)
msgBox.setText(str2)
msgBox.setInformativeText('<b>Proceed?</b>')
msgBox.addButton(PyQt5.QtWidgets.QMessageBox.Yes)
msgBox.addButton(PyQt5.QtWidgets.QMessageBox.Cancel)
msgBox.setDefaultButton(PyQt5.QtWidgets.QMessageBox.Yes)
ret = msgBox.exec_()
# app.exit()
if ret == PyQt5.QtWidgets.QMessageBox.Cancel:
print("So long and thanks for all the fish.")
self.exit_gui()
#
# Now the part which extracts and re-works the template
#
# Unpack template
print('>---------------------<')
print('Extracting template...')
cmd = ['tar', '-xf', '/reg/g/cfel/cheetah/template.tar']
cfel_file.spawn_subprocess(cmd, wait=True)
print("Done")
# Fix permissions
print('>---------------------<')
print('Fixing permissions...')
cmd = ['chgrp', '-R', expt, 'cheetah/']
cfel_file.spawn_subprocess(cmd, wait=True)
cmd = ['chmod', '-R', 'g+w', 'cheetah/']
cfel_file.spawn_subprocess(cmd, wait=True)
print("Done")
# Place configuration into /res
# print, 'Placing configuration files into /res...'
# cmd = ['/reg/g/cfel/cheetah/cheetah-stable/bin/make-labrynth']
# cfel_file.spawn_subprocess(cmd, wait=True)
# print("Done")
# Modify gui/crawler.config
print('>---------------------<')
file = 'cheetah/gui/crawler.config'
print('Modifying ', file)
# Replace XTC directory with the correct location using sed
xtcsedstr = str.replace(xtcdir, '/', '\\/')
cmd = ["sed", "-i", "-r", "s/(xtcdir=).*/\\1" + xtcsedstr + "/", file]
cfel_file.spawn_subprocess(cmd, wait=True)
print('>-------------------------<')
cmd = ['cat', file]
cfel_file.spawn_subprocess(cmd, wait=True)
print('>-------------------------<')
# Modify process/process
file = 'cheetah/process/process'
print('Modifying ', file)
cmd = ["sed", "-i", "-r", "s/(expt=).*/\\1\"" + expt + "\"/", file]
cfel_file.spawn_subprocess(cmd, wait=True)
xtcsedstr = str.replace(xtcdir, '/', '\\/')
cmd = ["sed", "-i", "-r", "s/(XTCDIR=).*/\\1\"" + xtcsedstr + "\"/", file]
cfel_file.spawn_subprocess(cmd, wait=True)
h5sedstr = str.replace(userdir, '/', '\\/') + '\/hdf5'
cmd = ["sed", "-i", "-r", "s/(H5DIR=).*/\\1\"" + h5sedstr + "\"/", file]
cfel_file.spawn_subprocess(cmd, wait=True)
confsedstr = str.replace(userdir, '/', '\\/') + '\/process'
cmd = [
"sed", "-i", "-r", "s/(CONFIGDIR=).*/\\1\"" + confsedstr + "\"/", file
]
cfel_file.spawn_subprocess(cmd, wait=True)
print('>-------------------------<')
cmd = ['head', file]
cfel_file.spawn_subprocess(cmd, wait=True)
print('>-------------------------<')
print("Working directory: ")
print(os.getcwd() + '/cheetah')
#end extract_lcls_template()
#
# Dialog box to further modify the configuration files
# (Detector name, encoders, etc)
#
modify_LCLS_cheetah_config_files(self)
return
def modify_LCLS_cheetah_config_files(self):
# Dialog box
result, ok = gui_dialogs.configure_cheetah_lcls_gui.configure_cheetah_dialog(
)
if ok == False:
print('Configuration scripts remain unchanged')
return
# This is needed because on startup we are one directory up from cheetah/gui, but when running we are in cheetah/gui
dir = os.getcwd()
if "cheetah/gui" in dir:
prefix = '..'
else:
prefix = 'cheetah'
# Modify detector type string
print("Modifying detector type...")
detectorType = result["detectorType"]
for file in glob.iglob(prefix + '/process/*.ini'):
cmd = [
"sed", "-i", "-r",
"/^detectorType/s/(detectorType=).*/\\1" + detectorType + "/", file
]
cfel_file.spawn_subprocess(cmd, wait=True)
cmd = [
"sed", "-i", "-r",
"/^detectorName/s/(detectorName=).*/\\1" + detectorType + "/", file
]
cfel_file.spawn_subprocess(cmd, wait=True)
# Modify detector Z encoder string
print("Modifying detector Z encoder...")
detectorZencoder = result["detectorZEncoder"]
for file in glob.iglob(prefix + '/process/*.ini'):
cmd = [
"sed", "-i", "-r", "/^detectorZpvname/s/(detectorZpvname=).*/\\1" +
detectorZencoder + "/", file
]
cfel_file.spawn_subprocess(cmd, wait=True)
# Modify data source in psana.cfg
# Unfortunately we gave different detectors different fields so we have to do some guessing
print("Modifying data source...")
file = prefix + '/process/psana.cfg'
dataSource = result["dataSource"]
if 'cspad' in dataSource.lower():
print('Cspad = ', dataSource)
cmd = [
"sed", "-i", "-r",
"/^cspadSource0/s/(cspadSource0 = ).*/\\1DetInfo(" + dataSource +
")/", file
]
cfel_file.spawn_subprocess(cmd, wait=True)
if 'rayonix' in dataSource.lower():
print('Rayonix = ', dataSource)
cmd = [
"sed", "-i", "-r",
"/^rayonixSource0/s/(rayonixSource0 = ).*/\\1DetInfo(" +
dataSource + ")/", file
]
cfel_file.spawn_subprocess(cmd, wait=True)
if 'pnccd' in dataSource.lower():
print('pnCCD = ', dataSource)
cmd = [
"sed", "-i", "-r",
"/^pnccdSource0/s/(pnccdSource0 = ).*/\\1DetInfo(" + dataSource +
")/", file
]
cfel_file.spawn_subprocess(cmd, wait=True)
print("Done modifying configuration scripts")
return
def extract_euxfel_template(self):
# Deduce experiment number, etc using de-referenced paths
realdir = os.getcwd()
print('os.getcwd() = ', realdir)
# Now for some EuXFEL-specific directory stuff
# /gpfs/exfel/u/scratch/SPB/201701/p002017/anton/cheetah-test
# to
# /gpfs/exfel/exp/SPB/201701/p002017
ss = realdir.split('/')
ss = ss[1:]
print(ss)
instr = ss[4]
run = ss[5]
expt = ss[6]
tempdir = str.join('/', ss[7:])
ssnew = ss
ssnew[2] = 'exp'
ssnew[3:6] = ss[4:7]
absroot = '/' + str.join('/', ssnew[0:6])
absdir = absroot + '/scratch/' + tempdir
datadir = absroot + '/raw'
userdir = absdir + '/cheetah'
print('Deduced experiment information:')
print(' Sensed directory: ', realdir)
print(' Preferred path: ', absdir)
print(' Instrument: ', instr)
print(' Experiment: ', expt)
print(' Proposal round: ', run)
print(' Data directory: ', datadir)
print(' Output directory: ', userdir)
#
# QMessageBox for confirmation before proceeding
#
msgBox = PyQt5.QtWidgets.QMessageBox()
str1 = []
str1.append('<b>Instrument:</b> ' + str(instr))
str1.append('<b>Experiment:</b> ' + str(expt))
str1.append('<b>Data directory:</b> ' + str(datadir))
str1.append('<b>Output directory:</b> ' + str(userdir))
# str1.append('<b>Relative path:</b> '+ str(dir))
str2 = str.join('<br>', str1)
msgBox.setText(str2)
msgBox.setInformativeText('<b>Proceed?</b>')
msgBox.addButton(PyQt5.QtWidgets.QMessageBox.Yes)
msgBox.addButton(PyQt5.QtWidgets.QMessageBox.Cancel)
msgBox.setDefaultButton(PyQt5.QtWidgets.QMessageBox.Yes)
ret = msgBox.exec_()
if ret == PyQt5.QtWidgets.QMessageBox.Cancel:
print("So long and thanks for all the fish.")
self.exit_gui()
#
# Extract the template
#
# Unpack template
print('>---------------------<')
print('Extracting template...')
template = '/gpfs/cfel/cxi/common/public/development/cheetah/template.tar'
cmd = ['tar', '-xf', template]
cfel_file.spawn_subprocess(cmd, wait=True)
print("Done")
# Fix permissions
print('>---------------------<')
print('Fixing permissions...')
cmd = ['chmod', '-R', 'ug+w', 'cheetah/']
cfel_file.spawn_subprocess(cmd, wait=True)
cmd = [
'chmod', 'ug+x', 'cheetah/process/process',
'cheetah/process/hitfinder', 'cheetah/process/cheetahparallel.py',
'cheetah/calib/agipd/scripts/preprocess-dk'
]
cfel_file.spawn_subprocess(cmd, wait=True)
print("Done")
#
# Modify configuration files
#
# Modify gui/crawler.config
print('>---------------------<')
file = 'cheetah/gui/crawler.config'
print('Modifying ', file)
# Replace XTC directory with the correct location using sed
sedstr_data = str.replace(datadir, '/', '\\/')
cmd = ["sed", "-i", "-r", "s/(xtcdir=).*/\\1" + sedstr_data + "/", file]
cfel_file.spawn_subprocess(cmd, wait=True)
print('>-------------------------<')
cmd = ['cat', file]
cfel_file.spawn_subprocess(cmd, wait=True)
print('>-------------------------<')
# Modify process/process
file = 'cheetah/process/process'
print('Modifying ', file)
exptstr = instr + '/' + run + '/' + expt
sedstr_exp = str.replace(exptstr, '/', '\\/')
cmd = ["sed", "-i", "-r", "s/(expt=).*/\\1" + sedstr_exp + "/", file]
cfel_file.spawn_subprocess(cmd, wait=True)
print('>-------------------------<')
cmd = ['head', file]
cfel_file.spawn_subprocess(cmd, wait=True)
print('>-------------------------<')
# Modify calibration script template
file = 'cheetah/calib/agipd/scripts/preprocess-dk'
print('Modifying ', file)
sedstr_data = str.replace(absroot, '/', '\\/')
cmd = [
"sed", "-i", "-r", "s/(--input_dir ).*/\\1" + sedstr_data + "/", file
]
cfel_file.spawn_subprocess(cmd, wait=True)
print("Working directory: ")
print(os.getcwd() + '/cheetah')
#
# Dialog box to further modify the configuration files
# (Detector name, encoders, etc)
#
print("Test here")
modify_EuXFEL_cheetah_config_files(self)
return
def relabel_dataset(self):
# Simple dialog box: http: // www.tutorialspoint.com / pyqt / pyqt_qinputdialog_widget.htm
text, ok = PyQt5.QtWidgets.QInputDialog.getText(
self, 'Change dataset label', 'New label:')
if ok == False:
return
newlabel = str(text)
print('New label is: ', newlabel)
dataset_csv = cfel_file.csv_to_dict('datasets.csv')
# Label all selected runs
runs = self.selected_runs()
for i, run in enumerate(runs['run']):
# Format directory string
olddir = runs['directory'][i]
newdir = '---'
if olddir != '---':
if 'LCLS' in self.compute_location['location']:
newdir = 'r{:04d}'.format(int(run))
elif 'max-exfl' in self.compute_location['location']:
newdir = 'r{:04d}'.format(int(run))
else:
newdir = run
newdir += '-' + newlabel
# Update Dataset in table
table_row = runs['row'][i]
self.table.setItem(table_row, 1,
PyQt5.QtWidgets.QTableWidgetItem(newlabel))
self.table.setItem(table_row, 5,
PyQt5.QtWidgets.QTableWidgetItem(newdir))
# Update dataset file
if run in dataset_csv['Run']:
ds_indx = dataset_csv['Run'].index(run)
dataset_csv['DatasetID'][ds_indx] = newlabel
dataset_csv['Directory'][ds_indx] = newdir
else:
dataset_csv['Run'].append(run)
dataset_csv['DatasetID'].append(newlabel)
dataset_csv['Directory'].append(newdir)
dataset_csv['iniFile'].append('---')
dataset_csv['calibFile'].append('---')
# Rename the directory
if olddir != '---':
if os.path.exists(self.config['hdf5dir'] + '/' + olddir):
try:
cmdarr = [
'mv', self.config['hdf5dir'] + '/' + olddir,
self.config['hdf5dir'] + '/' + newdir
]
cfel_file.spawn_subprocess(cmdarr)
except:
pass
if os.path.exists('../indexing/' + olddir):
try:
cmdarr = [
'mv', '../indexing/' + olddir,
'../indexing/' + newdir
]
cfel_file.spawn_subprocess(cmdarr)
except:
pass
# Sort dataset file to keep it in order
# Save datasets file
keys_to_save = [
'Run', 'DatasetID', 'Directory', 'iniFile', 'calibFile'
]
cfel_file.dict_to_csv('datasets.csv', dataset_csv, keys_to_save)
def modify_cheetah_config_files(self):
# Dialog box
result, ok = gui_dialogs.configure_cheetah_lcls_gui.configure_cheetah_dialog()
if ok == False:
print('Configuration scripts remain unchanged')
return
# This is needed because on startup we are one directory up from cheetah/gui, but when running we are in cheetah/gui
dir = os.getcwd()
if "cheetah/gui" in dir:
prefix = '..'
else:
prefix = 'cheetah'
# Modify detector type string
print("Modifying detector type...")
detectorType = result["detectorType"]
for file in glob.iglob(prefix+'/process/*.ini'):
cmd = ["sed", "-i", "-r", "/^detectorType/s/(detectorType=).*/\\1" + detectorType + "/", file]
cfel_file.spawn_subprocess(cmd, wait=True)
cmd = ["sed", "-i", "-r", "/^detectorName/s/(detectorName=).*/\\1" + detectorType + "/", file]
cfel_file.spawn_subprocess(cmd, wait=True)
# Modify detector Z encoder string
print("Modifying detector Z encoder...")
detectorZencoder = result["detectorZEncoder"]
for file in glob.iglob(prefix+'/process/*.ini'):
cmd = ["sed", "-i", "-r", "/^detectorZpvname/s/(detectorZpvname=).*/\\1" + detectorZencoder + "/", file]
cfel_file.spawn_subprocess(cmd, wait=True)
# Modify data source in psana.cfg
# Unfortunately we gave different detectors different fields so we have to do some guessing
print("Modifying data source...")
file = prefix+'/process/psana.cfg'
dataSource = result["dataSource"]
if 'cspad' in dataSource.lower():
print('Cspad = ', dataSource)
cmd = ["sed", "-i", "-r", "/^cspadSource0/s/(cspadSource0 = ).*/\\1DetInfo(" + dataSource + ")/", file]
cfel_file.spawn_subprocess(cmd, wait=True)
if 'rayonix' in dataSource.lower():
print('Rayonix = ', dataSource)
cmd = ["sed", "-i", "-r", "/^rayonixSource0/s/(rayonixSource0 = ).*/\\1DetInfo(" + dataSource + ")/", file]
cfel_file.spawn_subprocess(cmd, wait=True)
if 'pnccd' in dataSource.lower():
print('pnCCD = ', dataSource)
cmd = ["sed", "-i", "-r", "/^pnccdSource0/s/(pnccdSource0 = ).*/\\1DetInfo(" + dataSource + ")/", file]
cfel_file.spawn_subprocess(cmd, wait=True)
print("Done modifying configuration scripts")
return
def extract_lcls_template(self):
# Deduce experiment number, etc using de-referenced paths
# Assumes the file path follows the pattern: /reg/d/psdm/cxi/cxij4915/scratch/...
realdir = os.getcwd()
# Now for some LCLS-specific stuff
ss = realdir.split('/')
ss = ss[1:]
ss[1] = 'd'
ss[2] = 'psdm'
instr = ss[3]
expt = ss[4]
xtcdir = '/' + str.join('/', ss[0:5]) + '/xtc'
userdir = '/' + str.join('/', ss) + '/cheetah'
print('Deduced experiment information:')
print(' Relative path: ', dir)
print(' Absolute path: ', realdir)
print(' Instrument: ', instr)
print(' Experiment: ', expt)
print(' XTC directory: ', xtcdir)
print(' Output directory: ', userdir)
#
# QMessageBox for confirmation before proceeding
#
msgBox = PyQt5.QtWidgets.QMessageBox()
str1 = []
str1.append('<b>Instrument:</b> ' + str(instr))
str1.append('<b>Experiment:</b> ' + str(expt))
str1.append('<b>XTC directory:</b> ' + str(xtcdir))
str1.append('<b>Output directory:</b> ' + str(userdir))
# str1.append('<b>Relative path:</b> '+ str(dir))
str2 = str.join('<br>', str1)
msgBox.setText(str2)
msgBox.setInformativeText('<b>Proceed?</b>')
msgBox.addButton(PyQt5.QtWidgets.QMessageBox.Yes)
msgBox.addButton(PyQt5.QtWidgets.QMessageBox.Cancel)
msgBox.setDefaultButton(PyQt5.QtWidgets.QMessageBox.Yes)
ret = msgBox.exec_();
# app.exit()
if ret == PyQt5.QtWidgets.QMessageBox.Cancel:
print("So long and thanks for all the fish.")
self.exit_gui()
#
# Now the part which extracts and re-works the template
#
# Unpack template
print('>---------------------<')
print('Extracting template...')
cmd = ['tar', '-xf', '/reg/g/cfel/cheetah/template.tar']
cfel_file.spawn_subprocess(cmd, wait=True)
print("Done")
# Fix permissions
print('>---------------------<')
print('Fixing permissions...')
cmd = ['chgrp', '-R', expt, 'cheetah/']
cfel_file.spawn_subprocess(cmd, wait=True)
cmd = ['chmod', '-R', 'g+w', 'cheetah/']
cfel_file.spawn_subprocess(cmd, wait=True)
print("Done")
# Place configuration into /res
# print, 'Placing configuration files into /res...'
# cmd = ['/reg/g/cfel/cheetah/cheetah-stable/bin/make-labrynth']
# cfel_file.spawn_subprocess(cmd, wait=True)
# print("Done")
# Modify gui/crawler.config
print('>---------------------<')
file = 'cheetah/gui/crawler.config'
print('Modifying ', file)
# Replace XTC directory with the correct location using sed
xtcsedstr = str.replace(xtcdir, '/', '\\/')
cmd = ["sed", "-i", "-r", "s/(xtcdir=).*/\\1" + xtcsedstr + "/", file]
cfel_file.spawn_subprocess(cmd, wait=True)
print('>-------------------------<')
cmd = ['cat', file]
cfel_file.spawn_subprocess(cmd, wait=True)
print('>-------------------------<')
# Modify process/process
file = 'cheetah/process/process'
print('Modifying ', file)
cmd = ["sed", "-i", "-r", "s/(expt=).*/\\1\"" + expt + "\"/", file]
cfel_file.spawn_subprocess(cmd, wait=True)
xtcsedstr = str.replace(xtcdir, '/', '\\/')
cmd = ["sed", "-i", "-r", "s/(XTCDIR=).*/\\1\"" + xtcsedstr + "\"/", file]
cfel_file.spawn_subprocess(cmd, wait=True)
h5sedstr = str.replace(userdir, '/', '\\/') + '\/hdf5'
cmd = ["sed", "-i", "-r", "s/(H5DIR=).*/\\1\"" + h5sedstr + "\"/", file]
cfel_file.spawn_subprocess(cmd, wait=True)
confsedstr = str.replace(userdir, '/', '\\/') + '\/process'
cmd = ["sed", "-i", "-r", "s/(CONFIGDIR=).*/\\1\"" + confsedstr + "\"/", file]
cfel_file.spawn_subprocess(cmd, wait=True)
print('>-------------------------<')
cmd = ['head', file]
cfel_file.spawn_subprocess(cmd, wait=True)
print('>-------------------------<')
print("Working directory: ")
print(os.getcwd() + '/cheetah')
#end extract_lcls_template()
#
# Modify the configuration files
#
modify_cheetah_config_files(self)
return
def run_cheetah(self):
# Find .ini files for dropdown list
inifile_list = []
for file in glob.iglob('../process/*.ini'):
basename = os.path.basename(file)
inifile_list.append(basename)
#inifile_list = ['test1.ini','test2.ini']
# Info needed for the dialog box
dialog_info = {
'inifile_list': inifile_list,
'lastini': self.lastini,
'lastcalib': self.lastcalib,
'lasttag': self.lasttag
}
# Dialog box for dataset label and ini file
gui, ok = gui_dialogs.run_cheetah_gui.cheetah_dialog(dialog_info)
# Exit if cancel was pressed
if ok == False:
return
# Extract values from return dict
dataset = gui['dataset']
inifile = gui['inifile']
calibfile = gui['calibfile']
self.lasttag = dataset
self.lastini = inifile
self.lastcalib = calibfile
try:
dataset_csv = cfel_file.csv_to_dict('datasets.csv')
except:
print('Error occured reading datasets.csv (blank file?)')
print('Check file contents. Will return and do nothing.')
return
# Failing to read the dataset file looses all information (bad)
if len(dataset_csv['DatasetID']) is 0:
print("Error reading datasets.csv (blank file)")
print("Try again...")
return
if 'calibFile' not in dataset_csv.keys():
print('Adding calibFile to datasets.csv')
dataset_csv['calibFile'] = dataset_csv['iniFile']
# Process all selected runs
runs = self.selected_runs()
for i, run in enumerate(runs['run']):
print('------------ Start Cheetah process script ------------')
cmdarr = [self.config['process'], run, inifile, calibfile, dataset]
cfel_file.spawn_subprocess(cmdarr, shell=True)
# Format output directory string
# This clumsily selects between using run numbers and using directory names
# Need to fix this up sometime
print("Location: ", self.compute_location['location'])
if 'LCLS' in self.compute_location['location']:
dir = 'r{:04d}'.format(int(run))
elif 'max-exfl' in self.compute_location['location']:
dir = 'r{:04d}'.format(int(run))
elif 'max-cfel' in self.compute_location['location']:
dir = 'r{:04d}'.format(int(run))
else:
dir = run
dir += '-' + dataset
print('Output directory: ', dir)
#Update Dataset and Cheetah status in table
table_row = runs['row'][i]
self.table.setItem(table_row, 1,
PyQt5.QtWidgets.QTableWidgetItem(dataset))
self.table.setItem(table_row, 3,
PyQt5.QtWidgets.QTableWidgetItem('Submitted'))
self.table.setItem(table_row, 5,
PyQt5.QtWidgets.QTableWidgetItem(dir))
self.table.setItem(table_row, 10,
PyQt5.QtWidgets.QTableWidgetItem(inifile))
self.table.setItem(table_row, 11,
PyQt5.QtWidgets.QTableWidgetItem(calibfile))
self.table.item(table_row,
3).setBackground(PyQt5.QtGui.QColor(255, 255, 100))
# Update dataset file
if run in dataset_csv['Run']:
ds_indx = dataset_csv['Run'].index(run)
dataset_csv['DatasetID'][ds_indx] = dataset
dataset_csv['Directory'][ds_indx] = dir
dataset_csv['iniFile'][ds_indx] = inifile
dataset_csv['calibFile'][ds_indx] = calibfile
else:
dataset_csv['Run'].append(run)
dataset_csv['DatasetID'].append(dataset)
dataset_csv['Directory'].append(dir)
dataset_csv['iniFile'].append(inifile)
dataset_csv['calibFile'].append(calibfile)
print('------------ Finish Cheetah process script ------------')
# Sort dataset file to keep it in order
# Save datasets file
keys_to_save = [
'Run', 'DatasetID', 'Directory', 'iniFile', 'calibFile'
]
cfel_file.dict_to_csv('datasets.csv', dataset_csv, keys_to_save)
