def crystfel_viewindex_pick(self): file = cfel_file.dialog_pickfile(path='../indexing/streams', filter='*.stream', qtmainwin=self) if file != '': cmdarr = ['cxiview', '-s', file] cfel_file.spawn_subprocess(cmdarr)
def show_peakogram(self): runs = self.selected_runs() if len(runs['run']) == 0: return pkfile = runs['path'][0] + 'peaks.txt' cmdarr = ['peakogram.py', '-i', pkfile] cfel_file.spawn_subprocess(cmdarr)
def start_crawler(self): cmdarr = [ 'cheetah-crawler.py', '-l', self.compute_location['location'], '-d', self.config['xtcdir'], '-c', self.config['hdf5dir'], '-i', '../indexing/' ] cfel_file.spawn_subprocess(cmdarr) #, shell=True)
def crystfel_cellexplore_pick(self): file = cfel_file.dialog_pickfile(path='../indexing/streams', filter='*.stream', qtmainwin=self) if file != '': cmdarr = ['cell_explorer', file] cfel_file.spawn_subprocess(cmdarr)
def copy_darkcal(self): runs = self.selected_runs() if len(runs['run']) == 0: return for path in runs['path']: path += '*detector0-darkcal.h5' for dkcal in glob.iglob(path): cmdarr = ['cp', dkcal, '../calib/darkcal/.'] cfel_file.spawn_subprocess(cmdarr)
def set_current_badpix(self): file = cfel_file.dialog_pickfile(path='../calib/mask', filter='*.h5', qtmainwin=self) if file != '': file = os.path.relpath(file) basename = os.path.basename(file) dirname = os.path.dirname(file) cmdarr = ['ln', '-fs', basename, '../calib/mask/current-badpix.h5'] cfel_file.spawn_subprocess(cmdarr)
def crystfel_cellexplore(self): runs = self.selected_runs() if len(runs['run']) == 0: return dir = '../indexing/' + runs['directory'][0] stream = glob.glob(dir + '/*.stream') if len(stream) is 0: return cmdarr = ['cell_explorer', stream[0]] cfel_file.spawn_subprocess(cmdarr)
def set_cxiview_geom(self): file = cfel_file.dialog_pickfile(path='../calib/geometry', filter='*.geom', qtmainwin=self) if file != '': file = os.path.relpath(file) basename = os.path.basename(file) dirname = os.path.dirname(file) cmdarr = [ 'ln', '-fs', basename, '../calib/geometry/current-geometry.geom' ] cfel_file.spawn_subprocess(cmdarr)
def run_XFEL_detectorcalibration(self): runs = self.selected_runs() if len(runs) is 0: print('No runs selected') return for i, run in enumerate(runs['run']): print( '------------ Start XFEL detector calibration script ------------' ) #cmdarr = [self.config['process'], run] cmdarr = ["../process/calibrate_euxfel.sh", run] cfel_file.spawn_subprocess(cmdarr, shell=True)
def show_selected_images(self, filepat, field='data/data'): runs = self.selected_runs() # No run selected if len(runs['run']) == 0: return file = runs['path'][0] + filepat # Display if some file matches pattern (avoids error when no match) if len(glob.glob(file)) != 0: cmdarr = [ 'cxiview.py', '-g', self.config['geometry'], '-e', field, '-i', file ] cfel_file.spawn_subprocess(cmdarr) else: print("File does not seem to exist:") print(file)
def maskmaker(self): print("Mask maker selected") runs = self.selected_runs() if len(runs['run']) == 0: return file = runs['path'][0] file += '*detector0-class0-sum.h5' field = 'data/data' if len(glob.glob(file)) != 0: file = glob.glob(file)[0] cmdarr = [ 'maskMakerGUI.py', '-g', self.config['geometry'], file, field ] cfel_file.spawn_subprocess(cmdarr) else: print("File does not seem to exist:") print(file)
def set_darkrun(self): # This bit repeats copy_darkcal, but allows only one run and remembers filename for symlink runs = self.selected_runs() if len(runs['run']) == 0: return if len(runs['run']) > 1: print('Can only select one run') return # Copy files path = runs['path'][0] path += '*detector0-darkcal.h5' for dkcal in glob.iglob(path): cmdarr = ['cp', dkcal, '../calib/darkcal/.'] cfel_file.spawn_subprocess(cmdarr) # Create symbolic link file = os.path.relpath(dkcal) basename = os.path.basename(file) cmdarr = ['ln', '-fs', basename, '../calib/darkcal/current-darkcal.h5'] cfel_file.spawn_subprocess(cmdarr)
def modify_EuXFEL_cheetah_config_files(self): # Dialog box result, ok = gui_dialogs.configure_cheetah_euxfel_gui.configure_cheetah_dialog( ) if ok == False: print('Configuration scripts remain unchanged') return # This is needed because on startup we are one directory up from cheetah/gui, but when running we are in cheetah/gui dir = os.getcwd() if "cheetah/gui" in dir: prefix = '..' else: prefix = 'cheetah' print(result) detectorType = result["detectorType"] detectorName = result["detectorName"] # Modify detector type and name in .ini files print("Modifying detector type and name...") for file in glob.iglob(prefix + '/process/*.ini'): cmd = [ "sed", "-i", "-r", "/^detectorType/s/(detectorType=).*/\\1" + detectorType + "/", file ] cfel_file.spawn_subprocess(cmd, wait=True) cmd = [ "sed", "-i", "-r", "/^detectorName/s/(detectorName=).*/\\1" + detectorName + "/", file ] cfel_file.spawn_subprocess(cmd, wait=True) # Modify detector name in AGIPD preprocess-dk script file = prefix + '/calib/agipd/scripts/preprocess-dk' print('Modifying ', file) cmd = [ "sed", "-i", "-r", "s/(--detector_string ).*/\\1" + detectorName + "/", file ] cfel_file.spawn_subprocess(cmd, wait=True) return
def extract_euxfel_template(self): # Deduce experiment number, etc using de-referenced paths realdir = os.getcwd() print('os.getcwd() = ', realdir) # Now for some EuXFEL-specific directory stuff # /gpfs/exfel/u/scratch/SPB/201701/p002017/anton/cheetah-test # to # /gpfs/exfel/exp/SPB/201701/p002017 ss = realdir.split('/') ss = ss[1:] print(ss) instr = ss[4] run = ss[5] expt = ss[6] tempdir = str.join('/', ss[7:]) ssnew = ss ssnew[2] = 'exp' ssnew[3:6] = ss[4:7] absroot = '/' + str.join('/', ssnew[0:6]) absdir = absroot + '/scratch/' + tempdir datadir = absroot + '/raw' userdir = absdir + '/cheetah' print('Deduced experiment information:') print(' Sensed directory: ', realdir) print(' Preferred path: ', absdir) print(' Instrument: ', instr) print(' Experiment: ', expt) print(' Proposal round: ', run) print(' Data directory: ', datadir) print(' Output directory: ', userdir) # # QMessageBox for confirmation before proceeding # msgBox = PyQt5.QtWidgets.QMessageBox() str1 = [] str1.append('<b>Instrument:</b> ' + str(instr)) str1.append('<b>Experiment:</b> ' + str(expt)) str1.append('<b>Data directory:</b> ' + str(datadir)) str1.append('<b>Output directory:</b> ' + str(userdir)) # str1.append('<b>Relative path:</b> '+ str(dir)) str2 = str.join('<br>', str1) msgBox.setText(str2) msgBox.setInformativeText('<b>Proceed?</b>') msgBox.addButton(PyQt5.QtWidgets.QMessageBox.Yes) msgBox.addButton(PyQt5.QtWidgets.QMessageBox.Cancel) msgBox.setDefaultButton(PyQt5.QtWidgets.QMessageBox.Yes) ret = msgBox.exec_() if ret == PyQt5.QtWidgets.QMessageBox.Cancel: print("So long and thanks for all the fish.") self.exit_gui() # # Extract the template # # Unpack template print('>---------------------<') print('Extracting template...') template = '/gpfs/cfel/cxi/common/public/development/cheetah/template.tar' cmd = ['tar', '-xf', template] cfel_file.spawn_subprocess(cmd, wait=True) print("Done") # Fix permissions print('>---------------------<') print('Fixing permissions...') cmd = ['chmod', '-R', 'ug+w', 'cheetah/'] cfel_file.spawn_subprocess(cmd, wait=True) cmd = ['chmod', 'ug+x', 'cheetah/process/process', 'cheetah/process/hitfinder'] cfel_file.spawn_subprocess(cmd, wait=True) print("Done") # # Modify configuration files # # Modify gui/crawler.config print('>---------------------<') file = 'cheetah/gui/crawler.config' print('Modifying ', file) # Replace XTC directory with the correct location using sed xtcsedstr = str.replace(datadir, '/', '\\/') cmd = ["sed", "-i", "-r", "s/(xtcdir=).*/\\1" + xtcsedstr + "/", file] cfel_file.spawn_subprocess(cmd, wait=True) print('>-------------------------<') cmd = ['cat', file] cfel_file.spawn_subprocess(cmd, wait=True) print('>-------------------------<') # Modify process/process file = 'cheetah/process/process' print('Modifying ', file) exptstr = instr+'/'+run+'/'+expt expsedstr = str.replace(exptstr, '/', '\\/') cmd = ["sed", "-i", "-r", "s/(expt=).*/\\1" + expsedstr + "/", file] cfel_file.spawn_subprocess(cmd, wait=True) print('>-------------------------<') cmd = ['head', file] cfel_file.spawn_subprocess(cmd, wait=True) print('>-------------------------<') print("Working directory: ") print(os.getcwd() + '/cheetah') return
def index_runs(guiself, dirs=None, nocell=False, geopt=False): # Just some info print("Will process the following directories:") print(dirs) # Select geometry file and remember it geomfile = guiself.lastgeom if geomfile is None: geomfile = cfel_file.dialog_pickfile(path='../calib/geometry', filter='*.geom', qtmainwin=guiself) if geomfile is '': return guiself.lastgeom = geomfile # Default unit cell file cell = guiself.lastcell # This bit handles which recipe is selected # (including nocell and geopt options) recipe = guiself.lastindex if nocell: recipe = '../process/index_nocell.sh' if geopt: recipe = '../process/index_geopt.sh' # Launch dialog box for CrystFEL options dialog_in = { 'pdb_files': glob.glob('../calib/pdb/*.pdb') + glob.glob('../calib/pdb/*.cell'), 'geom_files': glob.glob('../calib/geometry/*.geom'), 'recipe_files': glob.glob('../process/index*.sh'), 'default_geom': geomfile, 'default_cell': cell, 'default_recipe': recipe } if dialog_in['default_cell'] is None or not cell in dialog_in['pdb_files']: dialog_in['default_cell'] = dialog_in['pdb_files'][0] dialog_out, ok = gui_dialogs.run_crystfel_dialog.dialog_box(dialog_in) # Exit if cancel was pressed if ok == False: return # Remember selections for later pdbfile = dialog_out['pdbfile'] geomfile = dialog_out['geomfile'] recipefile = dialog_out['recipefile'] guiself.lastcell = pdbfile guiself.lastgeom = geomfile if not nocell and not geopt: guiself.lastindex = recipefile #geomfile = os.path.abspath(geomfile) # # Loop through selected directories # Much of this repeats what is in index-nolatt.... simplify later # for dirbase in dirs: # Data location and destination location print(dir) h5dir = "../hdf5/" + dirbase indexdir = "../indexing/" + dirbase # Remove contents of any existing directory then recreate directory shutil.rmtree(indexdir, ignore_errors=True) os.makedirs(indexdir, exist_ok=True) # Use find to create file list cmd = 'find ' + os.path.abspath( h5dir) + ' -name \*.cxi > ' + indexdir + '/files.lst' print(cmd) os.system(cmd) # Copy scripts and calibrations to target directory cmdarr = ['cp', recipefile, indexdir + '/.'] cfel_file.spawn_subprocess(cmdarr, wait=True) cmdarr = ['cp', geomfile, indexdir + '/.'] cfel_file.spawn_subprocess(cmdarr, wait=True) cmdarr = ['cp', pdbfile, indexdir + '/.'] cfel_file.spawn_subprocess(cmdarr, wait=True) # Send indexing command to batch farm qlabel = 'indx-' + dirbase[1:5] logfile = 'bsub.log' abspath = os.path.abspath(indexdir) + '/' #Version 0 #bsub_cmd = ['bsub', '-q', 'psanaq', '-x', '-J', qlabel, '-o', logfile, '-cwd', abspath, 'source', './'+os.path.basename(recipefile), dirbase, os.path.basename(pdbfile), os.path.basename(geomfile)] #Version 1 bsub_cmd = [ '../process/queue.sh', qlabel, abspath, 'source', './' + os.path.basename(recipefile), dirbase, os.path.basename(pdbfile), os.path.basename(geomfile) ] # Submit it cfel_file.spawn_subprocess(bsub_cmd) print(">------------------------------<") return
def index_runs(guiself, dirs=None, nocell=False, geopt=False): # Just some info print("Will process the following directories:") print(dirs) # Select geometry file and remember it geomfile = guiself.lastgeom if geomfile is None: geomfile = cfel_file.dialog_pickfile(path='../calib/geometry', filter='*.geom', qtmainwin=guiself) if geomfile is '': return guiself.lastgeom = geomfile # Default unit cell file cell=guiself.lastcell # This bit handles which recipe is selected # (including nocell and geopt options) recipe = guiself.lastindex if nocell: recipe = '../process/index_nocell.sh' if geopt: recipe = '../process/index_geopt.sh' # Launch dialog box for CrystFEL options dialog_in = { 'pdb_files' : glob.glob('../calib/pdb/*.pdb')+glob.glob('../calib/pdb/*.cell'), 'geom_files' : glob.glob('../calib/geometry/*.geom'), 'recipe_files' : glob.glob('../process/index*.sh'), 'default_geom' : geomfile, 'default_cell' : cell, 'default_recipe' : recipe } if dialog_in['default_cell'] is None or not cell in dialog_in['pdb_files']: dialog_in['default_cell'] = dialog_in['pdb_files'][0] dialog_out, ok = gui_dialogs.run_crystfel_dialog.dialog_box(dialog_in) # Exit if cancel was pressed if ok == False: return # Remember selections for later pdbfile = dialog_out['pdbfile'] geomfile = dialog_out['geomfile'] recipefile = dialog_out['recipefile'] guiself.lastcell = pdbfile guiself.lastgeom = geomfile if not nocell and not geopt: guiself.lastindex = recipefile #geomfile = os.path.abspath(geomfile) # # Loop through selected directories # Much of this repeats what is in index-nolatt.... simplify later # for dirbase in dirs: # Data location and destination location print(dir) h5dir = "../hdf5/" + dirbase indexdir = "../indexing/" + dirbase # Remove contents of any existing directory then recreate directory shutil.rmtree(indexdir, ignore_errors=True) os.makedirs(indexdir, exist_ok=True) # Use find to create file list cmd = 'find ' + os.path.abspath(h5dir) + ' -name \*.cxi > ' + indexdir + '/files.lst' print(cmd) os.system(cmd) # Copy scripts and calibrations to target directory cmdarr = ['cp', recipefile , indexdir + '/.'] cfel_file.spawn_subprocess(cmdarr, wait=True) cmdarr = ['cp', geomfile, indexdir + '/.'] cfel_file.spawn_subprocess(cmdarr, wait=True) cmdarr = ['cp', pdbfile, indexdir + '/.'] cfel_file.spawn_subprocess(cmdarr, wait=True) # Send indexing command to batch farm qlabel = 'indx-'+dirbase[1:5] logfile = 'bsub.log' abspath = os.path.abspath(indexdir)+'/' #Version 0 #bsub_cmd = ['bsub', '-q', 'psanaq', '-x', '-J', qlabel, '-o', logfile, '-cwd', abspath, 'source', './'+os.path.basename(recipefile), dirbase, os.path.basename(pdbfile), os.path.basename(geomfile)] #Version 1 bsub_cmd = ['../process/queue.sh', qlabel, abspath, 'source', './'+os.path.basename(recipefile), dirbase, os.path.basename(pdbfile), os.path.basename(geomfile)] # Submit it cfel_file.spawn_subprocess(bsub_cmd) print(">------------------------------<") return
def extract_lcls_template(self): # Deduce experiment number, etc using de-referenced paths # Assumes the file path follows the pattern: /reg/d/psdm/cxi/cxij4915/scratch/... realdir = os.getcwd() # Now for some LCLS-specific stuff ss = realdir.split('/') ss = ss[1:] ss[1] = 'd' ss[2] = 'psdm' instr = ss[3] expt = ss[4] xtcdir = '/' + str.join('/', ss[0:5]) + '/xtc' userdir = '/' + str.join('/', ss) + '/cheetah' print('Deduced experiment information:') print(' Relative path: ', dir) print(' Absolute path: ', realdir) print(' Instrument: ', instr) print(' Experiment: ', expt) print(' XTC directory: ', xtcdir) print(' Output directory: ', userdir) # # QMessageBox for confirmation before proceeding # msgBox = PyQt5.QtWidgets.QMessageBox() str1 = [] str1.append('<b>Instrument:</b> ' + str(instr)) str1.append('<b>Experiment:</b> ' + str(expt)) str1.append('<b>XTC directory:</b> ' + str(xtcdir)) str1.append('<b>Output directory:</b> ' + str(userdir)) # str1.append('<b>Relative path:</b> '+ str(dir)) str2 = str.join('<br>', str1) msgBox.setText(str2) msgBox.setInformativeText('<b>Proceed?</b>') msgBox.addButton(PyQt5.QtWidgets.QMessageBox.Yes) msgBox.addButton(PyQt5.QtWidgets.QMessageBox.Cancel) msgBox.setDefaultButton(PyQt5.QtWidgets.QMessageBox.Yes) ret = msgBox.exec_() # app.exit() if ret == PyQt5.QtWidgets.QMessageBox.Cancel: print("So long and thanks for all the fish.") self.exit_gui() # # Now the part which extracts and re-works the template # # Unpack template print('>---------------------<') print('Extracting template...') cmd = ['tar', '-xf', '/reg/g/cfel/cheetah/template.tar'] cfel_file.spawn_subprocess(cmd, wait=True) print("Done") # Fix permissions print('>---------------------<') print('Fixing permissions...') cmd = ['chgrp', '-R', expt, 'cheetah/'] cfel_file.spawn_subprocess(cmd, wait=True) cmd = ['chmod', '-R', 'g+w', 'cheetah/'] cfel_file.spawn_subprocess(cmd, wait=True) print("Done") # Place configuration into /res # print, 'Placing configuration files into /res...' # cmd = ['/reg/g/cfel/cheetah/cheetah-stable/bin/make-labrynth'] # cfel_file.spawn_subprocess(cmd, wait=True) # print("Done") # Modify gui/crawler.config print('>---------------------<') file = 'cheetah/gui/crawler.config' print('Modifying ', file) # Replace XTC directory with the correct location using sed xtcsedstr = str.replace(xtcdir, '/', '\\/') cmd = ["sed", "-i", "-r", "s/(xtcdir=).*/\\1" + xtcsedstr + "/", file] cfel_file.spawn_subprocess(cmd, wait=True) print('>-------------------------<') cmd = ['cat', file] cfel_file.spawn_subprocess(cmd, wait=True) print('>-------------------------<') # Modify process/process file = 'cheetah/process/process' print('Modifying ', file) cmd = ["sed", "-i", "-r", "s/(expt=).*/\\1\"" + expt + "\"/", file] cfel_file.spawn_subprocess(cmd, wait=True) xtcsedstr = str.replace(xtcdir, '/', '\\/') cmd = ["sed", "-i", "-r", "s/(XTCDIR=).*/\\1\"" + xtcsedstr + "\"/", file] cfel_file.spawn_subprocess(cmd, wait=True) h5sedstr = str.replace(userdir, '/', '\\/') + '\/hdf5' cmd = ["sed", "-i", "-r", "s/(H5DIR=).*/\\1\"" + h5sedstr + "\"/", file] cfel_file.spawn_subprocess(cmd, wait=True) confsedstr = str.replace(userdir, '/', '\\/') + '\/process' cmd = [ "sed", "-i", "-r", "s/(CONFIGDIR=).*/\\1\"" + confsedstr + "\"/", file ] cfel_file.spawn_subprocess(cmd, wait=True) print('>-------------------------<') cmd = ['head', file] cfel_file.spawn_subprocess(cmd, wait=True) print('>-------------------------<') print("Working directory: ") print(os.getcwd() + '/cheetah') #end extract_lcls_template() # # Dialog box to further modify the configuration files # (Detector name, encoders, etc) # modify_LCLS_cheetah_config_files(self) return
def modify_LCLS_cheetah_config_files(self): # Dialog box result, ok = gui_dialogs.configure_cheetah_lcls_gui.configure_cheetah_dialog( ) if ok == False: print('Configuration scripts remain unchanged') return # This is needed because on startup we are one directory up from cheetah/gui, but when running we are in cheetah/gui dir = os.getcwd() if "cheetah/gui" in dir: prefix = '..' else: prefix = 'cheetah' # Modify detector type string print("Modifying detector type...") detectorType = result["detectorType"] for file in glob.iglob(prefix + '/process/*.ini'): cmd = [ "sed", "-i", "-r", "/^detectorType/s/(detectorType=).*/\\1" + detectorType + "/", file ] cfel_file.spawn_subprocess(cmd, wait=True) cmd = [ "sed", "-i", "-r", "/^detectorName/s/(detectorName=).*/\\1" + detectorType + "/", file ] cfel_file.spawn_subprocess(cmd, wait=True) # Modify detector Z encoder string print("Modifying detector Z encoder...") detectorZencoder = result["detectorZEncoder"] for file in glob.iglob(prefix + '/process/*.ini'): cmd = [ "sed", "-i", "-r", "/^detectorZpvname/s/(detectorZpvname=).*/\\1" + detectorZencoder + "/", file ] cfel_file.spawn_subprocess(cmd, wait=True) # Modify data source in psana.cfg # Unfortunately we gave different detectors different fields so we have to do some guessing print("Modifying data source...") file = prefix + '/process/psana.cfg' dataSource = result["dataSource"] if 'cspad' in dataSource.lower(): print('Cspad = ', dataSource) cmd = [ "sed", "-i", "-r", "/^cspadSource0/s/(cspadSource0 = ).*/\\1DetInfo(" + dataSource + ")/", file ] cfel_file.spawn_subprocess(cmd, wait=True) if 'rayonix' in dataSource.lower(): print('Rayonix = ', dataSource) cmd = [ "sed", "-i", "-r", "/^rayonixSource0/s/(rayonixSource0 = ).*/\\1DetInfo(" + dataSource + ")/", file ] cfel_file.spawn_subprocess(cmd, wait=True) if 'pnccd' in dataSource.lower(): print('pnCCD = ', dataSource) cmd = [ "sed", "-i", "-r", "/^pnccdSource0/s/(pnccdSource0 = ).*/\\1DetInfo(" + dataSource + ")/", file ] cfel_file.spawn_subprocess(cmd, wait=True) print("Done modifying configuration scripts") return
def extract_euxfel_template(self): # Deduce experiment number, etc using de-referenced paths realdir = os.getcwd() print('os.getcwd() = ', realdir) # Now for some EuXFEL-specific directory stuff # /gpfs/exfel/u/scratch/SPB/201701/p002017/anton/cheetah-test # to # /gpfs/exfel/exp/SPB/201701/p002017 ss = realdir.split('/') ss = ss[1:] print(ss) instr = ss[4] run = ss[5] expt = ss[6] tempdir = str.join('/', ss[7:]) ssnew = ss ssnew[2] = 'exp' ssnew[3:6] = ss[4:7] absroot = '/' + str.join('/', ssnew[0:6]) absdir = absroot + '/scratch/' + tempdir datadir = absroot + '/raw' userdir = absdir + '/cheetah' print('Deduced experiment information:') print(' Sensed directory: ', realdir) print(' Preferred path: ', absdir) print(' Instrument: ', instr) print(' Experiment: ', expt) print(' Proposal round: ', run) print(' Data directory: ', datadir) print(' Output directory: ', userdir) # # QMessageBox for confirmation before proceeding # msgBox = PyQt5.QtWidgets.QMessageBox() str1 = [] str1.append('<b>Instrument:</b> ' + str(instr)) str1.append('<b>Experiment:</b> ' + str(expt)) str1.append('<b>Data directory:</b> ' + str(datadir)) str1.append('<b>Output directory:</b> ' + str(userdir)) # str1.append('<b>Relative path:</b> '+ str(dir)) str2 = str.join('<br>', str1) msgBox.setText(str2) msgBox.setInformativeText('<b>Proceed?</b>') msgBox.addButton(PyQt5.QtWidgets.QMessageBox.Yes) msgBox.addButton(PyQt5.QtWidgets.QMessageBox.Cancel) msgBox.setDefaultButton(PyQt5.QtWidgets.QMessageBox.Yes) ret = msgBox.exec_() if ret == PyQt5.QtWidgets.QMessageBox.Cancel: print("So long and thanks for all the fish.") self.exit_gui() # # Extract the template # # Unpack template print('>---------------------<') print('Extracting template...') template = '/gpfs/cfel/cxi/common/public/development/cheetah/template.tar' cmd = ['tar', '-xf', template] cfel_file.spawn_subprocess(cmd, wait=True) print("Done") # Fix permissions print('>---------------------<') print('Fixing permissions...') cmd = ['chmod', '-R', 'ug+w', 'cheetah/'] cfel_file.spawn_subprocess(cmd, wait=True) cmd = [ 'chmod', 'ug+x', 'cheetah/process/process', 'cheetah/process/hitfinder', 'cheetah/process/cheetahparallel.py', 'cheetah/calib/agipd/scripts/preprocess-dk' ] cfel_file.spawn_subprocess(cmd, wait=True) print("Done") # # Modify configuration files # # Modify gui/crawler.config print('>---------------------<') file = 'cheetah/gui/crawler.config' print('Modifying ', file) # Replace XTC directory with the correct location using sed sedstr_data = str.replace(datadir, '/', '\\/') cmd = ["sed", "-i", "-r", "s/(xtcdir=).*/\\1" + sedstr_data + "/", file] cfel_file.spawn_subprocess(cmd, wait=True) print('>-------------------------<') cmd = ['cat', file] cfel_file.spawn_subprocess(cmd, wait=True) print('>-------------------------<') # Modify process/process file = 'cheetah/process/process' print('Modifying ', file) exptstr = instr + '/' + run + '/' + expt sedstr_exp = str.replace(exptstr, '/', '\\/') cmd = ["sed", "-i", "-r", "s/(expt=).*/\\1" + sedstr_exp + "/", file] cfel_file.spawn_subprocess(cmd, wait=True) print('>-------------------------<') cmd = ['head', file] cfel_file.spawn_subprocess(cmd, wait=True) print('>-------------------------<') # Modify calibration script template file = 'cheetah/calib/agipd/scripts/preprocess-dk' print('Modifying ', file) sedstr_data = str.replace(absroot, '/', '\\/') cmd = [ "sed", "-i", "-r", "s/(--input_dir ).*/\\1" + sedstr_data + "/", file ] cfel_file.spawn_subprocess(cmd, wait=True) print("Working directory: ") print(os.getcwd() + '/cheetah') # # Dialog box to further modify the configuration files # (Detector name, encoders, etc) # print("Test here") modify_EuXFEL_cheetah_config_files(self) return
def relabel_dataset(self): # Simple dialog box: http: // www.tutorialspoint.com / pyqt / pyqt_qinputdialog_widget.htm text, ok = PyQt5.QtWidgets.QInputDialog.getText( self, 'Change dataset label', 'New label:') if ok == False: return newlabel = str(text) print('New label is: ', newlabel) dataset_csv = cfel_file.csv_to_dict('datasets.csv') # Label all selected runs runs = self.selected_runs() for i, run in enumerate(runs['run']): # Format directory string olddir = runs['directory'][i] newdir = '---' if olddir != '---': if 'LCLS' in self.compute_location['location']: newdir = 'r{:04d}'.format(int(run)) elif 'max-exfl' in self.compute_location['location']: newdir = 'r{:04d}'.format(int(run)) else: newdir = run newdir += '-' + newlabel # Update Dataset in table table_row = runs['row'][i] self.table.setItem(table_row, 1, PyQt5.QtWidgets.QTableWidgetItem(newlabel)) self.table.setItem(table_row, 5, PyQt5.QtWidgets.QTableWidgetItem(newdir)) # Update dataset file if run in dataset_csv['Run']: ds_indx = dataset_csv['Run'].index(run) dataset_csv['DatasetID'][ds_indx] = newlabel dataset_csv['Directory'][ds_indx] = newdir else: dataset_csv['Run'].append(run) dataset_csv['DatasetID'].append(newlabel) dataset_csv['Directory'].append(newdir) dataset_csv['iniFile'].append('---') dataset_csv['calibFile'].append('---') # Rename the directory if olddir != '---': if os.path.exists(self.config['hdf5dir'] + '/' + olddir): try: cmdarr = [ 'mv', self.config['hdf5dir'] + '/' + olddir, self.config['hdf5dir'] + '/' + newdir ] cfel_file.spawn_subprocess(cmdarr) except: pass if os.path.exists('../indexing/' + olddir): try: cmdarr = [ 'mv', '../indexing/' + olddir, '../indexing/' + newdir ] cfel_file.spawn_subprocess(cmdarr) except: pass # Sort dataset file to keep it in order # Save datasets file keys_to_save = [ 'Run', 'DatasetID', 'Directory', 'iniFile', 'calibFile' ] cfel_file.dict_to_csv('datasets.csv', dataset_csv, keys_to_save)
def modify_cheetah_config_files(self): # Dialog box result, ok = gui_dialogs.configure_cheetah_lcls_gui.configure_cheetah_dialog() if ok == False: print('Configuration scripts remain unchanged') return # This is needed because on startup we are one directory up from cheetah/gui, but when running we are in cheetah/gui dir = os.getcwd() if "cheetah/gui" in dir: prefix = '..' else: prefix = 'cheetah' # Modify detector type string print("Modifying detector type...") detectorType = result["detectorType"] for file in glob.iglob(prefix+'/process/*.ini'): cmd = ["sed", "-i", "-r", "/^detectorType/s/(detectorType=).*/\\1" + detectorType + "/", file] cfel_file.spawn_subprocess(cmd, wait=True) cmd = ["sed", "-i", "-r", "/^detectorName/s/(detectorName=).*/\\1" + detectorType + "/", file] cfel_file.spawn_subprocess(cmd, wait=True) # Modify detector Z encoder string print("Modifying detector Z encoder...") detectorZencoder = result["detectorZEncoder"] for file in glob.iglob(prefix+'/process/*.ini'): cmd = ["sed", "-i", "-r", "/^detectorZpvname/s/(detectorZpvname=).*/\\1" + detectorZencoder + "/", file] cfel_file.spawn_subprocess(cmd, wait=True) # Modify data source in psana.cfg # Unfortunately we gave different detectors different fields so we have to do some guessing print("Modifying data source...") file = prefix+'/process/psana.cfg' dataSource = result["dataSource"] if 'cspad' in dataSource.lower(): print('Cspad = ', dataSource) cmd = ["sed", "-i", "-r", "/^cspadSource0/s/(cspadSource0 = ).*/\\1DetInfo(" + dataSource + ")/", file] cfel_file.spawn_subprocess(cmd, wait=True) if 'rayonix' in dataSource.lower(): print('Rayonix = ', dataSource) cmd = ["sed", "-i", "-r", "/^rayonixSource0/s/(rayonixSource0 = ).*/\\1DetInfo(" + dataSource + ")/", file] cfel_file.spawn_subprocess(cmd, wait=True) if 'pnccd' in dataSource.lower(): print('pnCCD = ', dataSource) cmd = ["sed", "-i", "-r", "/^pnccdSource0/s/(pnccdSource0 = ).*/\\1DetInfo(" + dataSource + ")/", file] cfel_file.spawn_subprocess(cmd, wait=True) print("Done modifying configuration scripts") return
def extract_lcls_template(self): # Deduce experiment number, etc using de-referenced paths # Assumes the file path follows the pattern: /reg/d/psdm/cxi/cxij4915/scratch/... realdir = os.getcwd() # Now for some LCLS-specific stuff ss = realdir.split('/') ss = ss[1:] ss[1] = 'd' ss[2] = 'psdm' instr = ss[3] expt = ss[4] xtcdir = '/' + str.join('/', ss[0:5]) + '/xtc' userdir = '/' + str.join('/', ss) + '/cheetah' print('Deduced experiment information:') print(' Relative path: ', dir) print(' Absolute path: ', realdir) print(' Instrument: ', instr) print(' Experiment: ', expt) print(' XTC directory: ', xtcdir) print(' Output directory: ', userdir) # # QMessageBox for confirmation before proceeding # msgBox = PyQt5.QtWidgets.QMessageBox() str1 = [] str1.append('<b>Instrument:</b> ' + str(instr)) str1.append('<b>Experiment:</b> ' + str(expt)) str1.append('<b>XTC directory:</b> ' + str(xtcdir)) str1.append('<b>Output directory:</b> ' + str(userdir)) # str1.append('<b>Relative path:</b> '+ str(dir)) str2 = str.join('<br>', str1) msgBox.setText(str2) msgBox.setInformativeText('<b>Proceed?</b>') msgBox.addButton(PyQt5.QtWidgets.QMessageBox.Yes) msgBox.addButton(PyQt5.QtWidgets.QMessageBox.Cancel) msgBox.setDefaultButton(PyQt5.QtWidgets.QMessageBox.Yes) ret = msgBox.exec_(); # app.exit() if ret == PyQt5.QtWidgets.QMessageBox.Cancel: print("So long and thanks for all the fish.") self.exit_gui() # # Now the part which extracts and re-works the template # # Unpack template print('>---------------------<') print('Extracting template...') cmd = ['tar', '-xf', '/reg/g/cfel/cheetah/template.tar'] cfel_file.spawn_subprocess(cmd, wait=True) print("Done") # Fix permissions print('>---------------------<') print('Fixing permissions...') cmd = ['chgrp', '-R', expt, 'cheetah/'] cfel_file.spawn_subprocess(cmd, wait=True) cmd = ['chmod', '-R', 'g+w', 'cheetah/'] cfel_file.spawn_subprocess(cmd, wait=True) print("Done") # Place configuration into /res # print, 'Placing configuration files into /res...' # cmd = ['/reg/g/cfel/cheetah/cheetah-stable/bin/make-labrynth'] # cfel_file.spawn_subprocess(cmd, wait=True) # print("Done") # Modify gui/crawler.config print('>---------------------<') file = 'cheetah/gui/crawler.config' print('Modifying ', file) # Replace XTC directory with the correct location using sed xtcsedstr = str.replace(xtcdir, '/', '\\/') cmd = ["sed", "-i", "-r", "s/(xtcdir=).*/\\1" + xtcsedstr + "/", file] cfel_file.spawn_subprocess(cmd, wait=True) print('>-------------------------<') cmd = ['cat', file] cfel_file.spawn_subprocess(cmd, wait=True) print('>-------------------------<') # Modify process/process file = 'cheetah/process/process' print('Modifying ', file) cmd = ["sed", "-i", "-r", "s/(expt=).*/\\1\"" + expt + "\"/", file] cfel_file.spawn_subprocess(cmd, wait=True) xtcsedstr = str.replace(xtcdir, '/', '\\/') cmd = ["sed", "-i", "-r", "s/(XTCDIR=).*/\\1\"" + xtcsedstr + "\"/", file] cfel_file.spawn_subprocess(cmd, wait=True) h5sedstr = str.replace(userdir, '/', '\\/') + '\/hdf5' cmd = ["sed", "-i", "-r", "s/(H5DIR=).*/\\1\"" + h5sedstr + "\"/", file] cfel_file.spawn_subprocess(cmd, wait=True) confsedstr = str.replace(userdir, '/', '\\/') + '\/process' cmd = ["sed", "-i", "-r", "s/(CONFIGDIR=).*/\\1\"" + confsedstr + "\"/", file] cfel_file.spawn_subprocess(cmd, wait=True) print('>-------------------------<') cmd = ['head', file] cfel_file.spawn_subprocess(cmd, wait=True) print('>-------------------------<') print("Working directory: ") print(os.getcwd() + '/cheetah') #end extract_lcls_template() # # Modify the configuration files # modify_cheetah_config_files(self) return
def run_cheetah(self): # Find .ini files for dropdown list inifile_list = [] for file in glob.iglob('../process/*.ini'): basename = os.path.basename(file) inifile_list.append(basename) #inifile_list = ['test1.ini','test2.ini'] # Info needed for the dialog box dialog_info = { 'inifile_list': inifile_list, 'lastini': self.lastini, 'lastcalib': self.lastcalib, 'lasttag': self.lasttag } # Dialog box for dataset label and ini file gui, ok = gui_dialogs.run_cheetah_gui.cheetah_dialog(dialog_info) # Exit if cancel was pressed if ok == False: return # Extract values from return dict dataset = gui['dataset'] inifile = gui['inifile'] calibfile = gui['calibfile'] self.lasttag = dataset self.lastini = inifile self.lastcalib = calibfile try: dataset_csv = cfel_file.csv_to_dict('datasets.csv') except: print('Error occured reading datasets.csv (blank file?)') print('Check file contents. Will return and do nothing.') return # Failing to read the dataset file looses all information (bad) if len(dataset_csv['DatasetID']) is 0: print("Error reading datasets.csv (blank file)") print("Try again...") return if 'calibFile' not in dataset_csv.keys(): print('Adding calibFile to datasets.csv') dataset_csv['calibFile'] = dataset_csv['iniFile'] # Process all selected runs runs = self.selected_runs() for i, run in enumerate(runs['run']): print('------------ Start Cheetah process script ------------') cmdarr = [self.config['process'], run, inifile, calibfile, dataset] cfel_file.spawn_subprocess(cmdarr, shell=True) # Format output directory string # This clumsily selects between using run numbers and using directory names # Need to fix this up sometime print("Location: ", self.compute_location['location']) if 'LCLS' in self.compute_location['location']: dir = 'r{:04d}'.format(int(run)) elif 'max-exfl' in self.compute_location['location']: dir = 'r{:04d}'.format(int(run)) elif 'max-cfel' in self.compute_location['location']: dir = 'r{:04d}'.format(int(run)) else: dir = run dir += '-' + dataset print('Output directory: ', dir) #Update Dataset and Cheetah status in table table_row = runs['row'][i] self.table.setItem(table_row, 1, PyQt5.QtWidgets.QTableWidgetItem(dataset)) self.table.setItem(table_row, 3, PyQt5.QtWidgets.QTableWidgetItem('Submitted')) self.table.setItem(table_row, 5, PyQt5.QtWidgets.QTableWidgetItem(dir)) self.table.setItem(table_row, 10, PyQt5.QtWidgets.QTableWidgetItem(inifile)) self.table.setItem(table_row, 11, PyQt5.QtWidgets.QTableWidgetItem(calibfile)) self.table.item(table_row, 3).setBackground(PyQt5.QtGui.QColor(255, 255, 100)) # Update dataset file if run in dataset_csv['Run']: ds_indx = dataset_csv['Run'].index(run) dataset_csv['DatasetID'][ds_indx] = dataset dataset_csv['Directory'][ds_indx] = dir dataset_csv['iniFile'][ds_indx] = inifile dataset_csv['calibFile'][ds_indx] = calibfile else: dataset_csv['Run'].append(run) dataset_csv['DatasetID'].append(dataset) dataset_csv['Directory'].append(dir) dataset_csv['iniFile'].append(inifile) dataset_csv['calibFile'].append(calibfile) print('------------ Finish Cheetah process script ------------') # Sort dataset file to keep it in order # Save datasets file keys_to_save = [ 'Run', 'DatasetID', 'Directory', 'iniFile', 'calibFile' ] cfel_file.dict_to_csv('datasets.csv', dataset_csv, keys_to_save)