def _get_ncores(self): # TODO: make Ncores (number of subjobs for current calculation) flexible: # * depending on urgency/priority # * depending on how busy the cluster is # * avoid that calculation time will depend on a single job that takes forever syscfg = system_configuration.getInstance() return syscfg['number of cores']
def __read_hlut_conf(fname=None): """ Use the ``configparser`` module to read the config file, then try to convert each section into a HLUT definition. The 'fname' argument should ONLY be different from None during unit tests. """ hlut_parser = configparser.ConfigParser() hlut_parser.optionxform = str # keys (CT protocol names) should be case sensitive syscfg = system_configuration.getInstance() if fname is None: fname = os.path.join(syscfg['CT'], "hlut.conf") if not os.path.exists(fname): raise RuntimeError( f"ERROR: cannot find the HLUT configuration file '{fname}'.") all_hluts = dict() with open(fname, "r") as fp: hlut_parser.read_file(fp) for s in hlut_parser.sections(): try: h = hlut(s, hlut_parser[s]) all_hluts[s] = h except Exception as e: logger.error( f"OOPSIE skipping section '{s}' in hlut.conf because: '{e}'" ) return hlut_conf(all_hluts)
def __init__(self,current): QtWidgets.QWidget.__init__(self) self.current = current logger.debug("start creating phantom details widget") syscfg = system_configuration.getInstance() self._phantoms = syscfg["phantom_defs"] logger.debug("creating PHANTOM widget (e.g. 'PMMA box' or 'WATER box')") self.phantomLabel = QtWidgets.QLabel("Phantom") self.phantomComboBox = QtWidgets.QComboBox(self) self.phantomComboBox.addItems(["choose phantom"]+[spec.gui_name for spec in self._phantoms.values()]) self.phantomComboBox.setCurrentIndex(0) self.phantomWidget = QtWidgets.QWidget() self.hBoxlayoutPHANTOM = QtWidgets.QHBoxLayout() self.hBoxlayoutPHANTOM.addWidget(self.phantomLabel) self.hBoxlayoutPHANTOM.addWidget(self.phantomComboBox) self.phantomWidget.setLayout(self.hBoxlayoutPHANTOM) self.vBoxlayoutPhantomDetails = QtWidgets.QVBoxLayout() self.vBoxlayoutPhantomDetails.addWidget(self.phantomWidget) self.setLayout(self.vBoxlayoutPhantomDetails) self.phantomComboBox.currentIndexChanged[int].connect(self.UpdatePhantomGEO) self.rp_filepath = self.current.rp_filepath self.current.Subscribe(self) logger.debug("finished creating phantom details widget")
def __init__(self, cfg): self.out_dose_nxyz = cfg.out_dose_nxyz.astype(np.int) self.sim_dose_nxyz = cfg.sim_dose_nxyz.astype(np.int) if bool(cfg.mask_mhd): self.mask = itk.imread(os.path.join(cfg.workdir, cfg.mask_mhd)) self.amask = itk.array_view_from_image(self.mask) if not (self.out_dose_nxyz[::-1] == self.amask.shape).all(): msg = "mask resolution {} inconsistent with expected output dose resolution {}".format( self.out_dose_nxyz[::-1], self.amask.shape) logger.error(msg) raise RuntimeError(msg) else: self.mask = None if bool(cfg.mass_mhd): #self.mass = itk.imread(cfg.mass_mhd) self.mass = itk.imread(os.path.join(cfg.workdir, cfg.mass_mhd)) self.amass = itk.array_view_from_image(self.mass) if not (self.sim_dose_nxyz[::-1] == self.amass.shape).all(): raise RuntimeError( "mass resolution {} inconsistent with expected simulation dose resolution {}" .format(self.sim_dose_nxyz[::-1], self.amass.shape)) else: self.mass = None if bool(cfg.mass_mhd) != bool(cfg.mask_mhd): msg = "got mass_mhd={} and mask_mhd={}".format( cfg.mass_mhd, cfg.mask_mhd) msg += "you should provide EITHER both the mass and the mask file, OR neither of them." logger.error(msg) raise RuntimeError(msg) self.cfg = cfg syscfg = system_configuration.getInstance() self.ntop = syscfg["n top voxels for mean dose max"] self.toppct = syscfg[ "dose threshold as fraction in percent of mean dose max"] self.reset()
def _launch_subjobs(self): if not os.path.isdir(self._RUNGATE_submit_directory): logger.error("cannot find submit directory {}".format( self._RUNGATE_submit_directory)) return -1 syscfg = system_configuration.getInstance() save_cwd = os.getcwd() os.chdir(self._RUNGATE_submit_directory) ymd_hms = time.strftime("%Y-%m-%d %H:%M:%S") userstuff = self.details.WriteUserSettings( self._qspecs, ymd_hms, self._RUNGATE_submit_directory) ret = os.system("condor_submit_dag ./RunGATE.dagman") if ret == 0: msg = "Job submitted at {}.\n".format(ymd_hms) msg += "User settings are summarized in \n'{}'\n".format(userstuff) self._summary += msg msg += "Final output will be saved in \n'{}'\n".format( self.details.output_job) msg += "GATE job submit directory:\n'{}'\n".format( self._RUNGATE_submit_directory) msg += "GUI logs:\n'{}'\n".format(syscfg["log file path"]) logger.info(msg) #success = launch_job_control_daemon(self._RUNGATE_submit_directory) #if self.details.mc_stat_type == MCStatType.Xpct_unc_in_target: ret = os.system( "{bindir}/job_control_daemon.py -l {username} -t {timeout} -n {minprim} -u {uncgoal} -p {poll} -d -w '{workdir}'" .format( bindir=syscfg['bindir'], username=syscfg['username'], # DONE: change this into Nprim, Unc, TimeOut settings #goal=self.details.mc_stat_thr, timeout=self.details.mc_stat_thr[ MCStatType.Nminutes_per_job], minprim=self.details.mc_stat_thr[ MCStatType.Nions_per_beam], uncgoal=self.details.mc_stat_thr[ MCStatType.Xpct_unc_in_target], poll=syscfg['stop on script actor time interval [s]'], workdir=self._RUNGATE_submit_directory)) if ret == 0: msg = "successful start of job statistics daemon" self._summary += msg + "\n" logger.info(msg) else: msg = "FAILED to start job statistics daemon" self._summary += msg + "\n" logger.error(msg) else: msg = "Job submit error: return value {}".format(ret) self._summary += msg logger.error(msg) os.chdir(save_cwd) return ret
def __init__(self, details): syscfg = system_configuration.getInstance() self._details = details self._ect = 0. self._badchars = re.compile("[^a-zA-Z0-9_]") self._time_stamp = time.strftime("%Y_%m_%d_%H_%M_%S") self._summary = str() self._summary += "TPS Plan/Beamset UID: " + details.uid + "\n" self._summary += "IDC user: {}\n".format(syscfg['username']) self._set_cleanup_policy(not syscfg['debug']) self._mac_files = [] self._qspecs = {} self._generate_RUNGATE_submit_directory() self._populate_RUNGATE_submit_directory()
def __init__(self,current): QtWidgets.QWidget.__init__(self) self.current = current logger.debug("creating ROI image details widget") self.rp_filepath = "" self.roitable = QtWidgets.QLabel("If you import a treatment plan, then you'll find a ROI table here") self.vBoxlayoutROIDetails = QtWidgets.QVBoxLayout() self.vBoxlayoutROIDetails.addWidget(self.roitable) self.setLayout(self.vBoxlayoutROIDetails) self.no_override = "(no override)" self.roinames=list() self.padding_due_to_dosegrid = False syscfg = system_configuration.getInstance() self.ct_override_material_list = [self.no_override] + sorted(syscfg['ct override list'].keys()) current.Subscribe(self)
def hlut_cache_dir(density,composition,HUtol,create=False): syscfg = system_configuration.getInstance() h4sh = hlut_hash(density,composition) cache_dir = os.path.join(syscfg['CT'],'cache',h4sh,str(HUtol)) if os.path.isdir(cache_dir): return cache_dir elif create: os.makedirs(cache_dir,exist_ok=True) # store original density and composition files cache_parent = os.path.dirname(cache_dir) # copying the input files so that you know to which protocol this cache dir corresponds shutil.copy(density,os.path.join(cache_parent,os.path.basename(density))) shutil.copy(composition,os.path.join(cache_parent,os.path.basename(composition))) return cache_dir # TODO: alternatively, throw something... return None
def get_hu2mat_files(self, hutol=None): if hutol is not None: self.hutol = hutol if self.type == "Schneider": self.cache_dir = hlut_cache_dir(self.density, self.composition, self.hutol, create=False) if self.cache_dir is None: # this will run Gate to run the db/hu2mat generation if necessary ok, self.cache_dir = generate_hlut_cache( self.density, self.composition, self.hutol) if not ok: raise RuntimeError( f"{self.name} failed to create HU2material tables in cache directory {self.cache_dir}." ) humatdb = os.path.join(self.cache_dir, 'patient-HUmaterials.db') hu2mattxt = os.path.join(self.cache_dir, 'patient-HU2mat.txt') elif self.type == "Commissioning": h4sh = hashlib.md5() for line in self.hu2mat_lines: h4sh.update(bytes(line, encoding='utf-8')) syscfg = system_configuration.getInstance() self.cache_dir = os.path.join(syscfg['CT/cache'], h4sh.hexdigest()) os.makedirs(self.cache_dir, exist_ok=True) hu2mattxt = os.path.join(self.cache_dir, 'commissioning-HU2mat.txt') humatdb = os.path.join(self.cache_dir, 'commissioning-HUmaterials.db') if not os.path.exists(hu2mattxt): with open(hu2mattxt, "w") as hu2matfp: # write hu2mat cache file hu2matfp.write("\n".join(self.hu2mat_lines)) hu2matfp.write("\n") if not os.path.exists(humatdb): with open(humatdb, "w"): # write empty file pass else: raise RuntimeError("OOPSIE: programming error") assert (os.path.exists(humatdb)) assert (os.path.exists(hu2mattxt)) return hu2mattxt, humatdb
def PlanUpdate(self): names = self.details.beam_names numbers = self.details.beam_numbers assert (len(numbers) == len(names)) oldNrow = self.beamSelector.rowCount() self.beamSelector.setRowCount(len(names)) self.beamSelector.setVerticalHeaderLabels(numbers) syscfg = system_configuration.getInstance() for i, name in enumerate(names): if i < oldNrow: self.beamSelector.item(i, 0).setText(name) beamCheck = self.beamButtonGroup.button(i) beamCheck.setCheckState(QtCore.Qt.Checked) else: self.beamSelector.setItem(i, 0, QtWidgets.QTableWidgetItem(name)) beamCheck = QtWidgets.QCheckBox() beamCheck.setCheckState(QtCore.Qt.Checked) beamWidget = QtWidgets.QWidget() beamLayout = QtWidgets.QHBoxLayout(beamWidget) beamLayout.addWidget(beamCheck) beamLayout.setAlignment(QtCore.Qt.AlignCenter) beamLayout.setContentsMargins(0, 0, 0, 0) beamWidget.setLayout(beamLayout) self.beamSelector.setCellWidget(i, 1, beamWidget) self.beamButtonGroup.addButton(beamCheck, i) for i in range(self.beamSelector.rowCount(), len(self.beamButtonGroup.buttons())): old_checkbox = self.beamButtonGroup.button(i) self.beamButtonGroup.removeButton(old_checkbox) self.spinboxNJobs.setValue(self.ncores) for imc in range(MCStatType.NTypes): mcmin, mcval, mcmax, mcstep, mcdef = syscfg[ MCStatType.cfglabels[imc]] self.spinboxes[imc].setValue(mcval) if mcdef: self.checkboxes[imc].setCheckState(QtCore.Qt.Checked)
def generate_hlut_cache(density,composition,HUtol,db=None): syscfg = system_configuration.getInstance() cache_dir = hlut_cache_dir(density,composition,HUtol,create=True) if db is None: materialsdb = os.path.join(syscfg['commissioning'],syscfg['materials database']) else: materialsdb = db hlut_gen_cache_mac = os.path.join(syscfg['config dir'],'hlut_gen_cache.mac') humatdb = os.path.join(cache_dir,'patient-HUmaterials.db') hu2mattxt = os.path.join(cache_dir,'patient-HU2mat.txt') adict = dict([("MATERIALS_DB", materialsdb), ("SCHNEIDER_COMPOSITION_FILE",composition), ("SCHNEIDER_DENSITY_FILE", density), ("DENSITY_TOLERANCE", HUtol), ("MATERIALS_INTERPOLATED", humatdb), ("HU2MAT_TABLE", hu2mattxt)]) aliases = "".join(["[{},{}]".format(name,val) for name,val in adict.items()]) gensh = os.path.join("/tmp","hlut_gen_cache.sh") tstart = datetime.now() gate_log = os.path.join(syscfg['logging'],tstart.strftime("hlut_gen_cache_%y_%m_%d_%H_%M_%S.log")) with open(gensh,"w") as gensh_fh: gensh_fh.write("#!/usr/bin/env bash\n") gensh_fh.write("set -e\n") gensh_fh.write("set -x\n") gensh_fh.write("source {}\n".format(syscfg['gate_env.sh'])) gensh_fh.write("time Gate -a{} {} >& {}\n".format(aliases,hlut_gen_cache_mac,gate_log)) os.chmod(gensh,stat.S_IREAD|stat.S_IRWXU) logger.info("generating cache for {} and {} with density tolerance {} g/cm3".format(density,composition,HUtol)) logger.info("cache dir: {}".format(cache_dir)) #ret=os.system( gensh + " >& " + gate_log ) ret=os.system( gensh ) tend=datetime.now() dbl_chk = os.path.exists(humatdb) and os.path.exists(hu2mattxt) logger.info("return code: {}, job took {} seconds, new HLUT cache files {} exist.".format(ret,(tend-tstart).total_seconds(),("DO" if dbl_chk else "DO NOT"))) logger.info("logs are in: {}".format(gate_log)) success = (ret==0) and dbl_chk return success, cache_dir
def __init__(self, details): QtWidgets.QWidget.__init__(self) self.details = details details.Subscribe(self) self.quantityButtonGroup = QtWidgets.QButtonGroup() self.quantityButtonGroup.setExclusive(True) self.vBoxLayoutIDCMCStatistics = QtWidgets.QVBoxLayout() self.quantitySelection = QtWidgets.QWidget() self.layoutQuantitySelection = QtWidgets.QHBoxLayout() self.spinboxes = list() self.checkboxes = list() syscfg = system_configuration.getInstance() self.ncores = syscfg['number of cores'] for imc in range(MCStatType.NTypes): mcmin, mcval, mcmax, mcstep, mcdef = syscfg[ MCStatType.cfglabels[imc]] labelMCStat = QtWidgets.QLabel(MCStatType.guilabels[imc]) checkboxMCStat = QtWidgets.QCheckBox(self) checkboxMCStat.setCheckState( QtCore.Qt.Checked if mcdef else QtCore.Qt.Unchecked) self.checkboxes.append(checkboxMCStat) self.quantityButtonGroup.addButton(checkboxMCStat, imc) if imc == MCStatType.Nminutes_per_job: spinboxMCStat = QtWidgets.QSpinBox(self) self.spinboxNJobs = QtWidgets.QSpinBox(self) self.spinboxNJobs.setMinimum(1) self.spinboxNJobs.setValue(self.ncores) self.spinboxNJobs.setMaximum(10000) self.spinboxNJobs.setSuffix(" jobs") self.spinboxNJobs.valueChanged[int].connect(self.UpdateNJobs) elif imc == MCStatType.Nions_per_beam: spinboxMCStat = QtWidgets.QSpinBox(self) else: spinboxMCStat = QtWidgets.QDoubleSpinBox(self) spinboxMCStat.setSuffix(" " + MCStatType.unit[imc]) self.spinboxes.append(spinboxMCStat) logger.debug("{}. {} min={} val={} max={} step={} def={}".format( imc, MCStatType.cfglabels[imc], mcmin, mcval, mcmax, mcstep, mcdef)) spinboxMCStat.setMinimum(mcmin) spinboxMCStat.setMaximum(mcmax) spinboxMCStat.setValue(mcval) spinboxMCStat.setSingleStep(mcstep) spinboxMCStat.setReadOnly(False) spinboxMCStat.setEnabled(mcdef) checkboxMCStat.toggled.connect(spinboxMCStat.setEnabled) if MCStatType.is_int[imc]: spinboxMCStat.valueChanged[int].connect( lambda ival: self.UpdateThresholdValue(ival, imc)) else: spinboxMCStat.valueChanged[float].connect( lambda fval: self.UpdateThresholdValue(fval, imc)) if mcdef: self.UpdateSelectedQuantity(imc) layoutSelectMCStat = QtWidgets.QVBoxLayout() layoutSelectMCStat.addWidget(labelMCStat) layoutSelectMCStat.addWidget(checkboxMCStat) layoutSelectMCStat.addWidget(spinboxMCStat) if imc == MCStatType.Nminutes_per_job: layoutSelectMCStat.addWidget(self.spinboxNJobs) selectMCStat = QtWidgets.QWidget() selectMCStat.setLayout(layoutSelectMCStat) self.layoutQuantitySelection.addWidget(selectMCStat) self.quantitySelection.setLayout(self.layoutQuantitySelection) self.vBoxLayoutIDCMCStatistics.addWidget(self.quantitySelection) self.beamSelector = QtWidgets.QTableWidget() self.beamSelector.setRowCount(0) self.beamSelector.setColumnCount(2) self.beamSelector.setHorizontalHeaderLabels( ["Name", "Included in simulation"]) self.beamButtonGroup = QtWidgets.QButtonGroup() self.beamButtonGroup.setExclusive(False) self.beamButtonGroup.buttonClicked[int].connect( self.UpdateBeamSelection) self.vBoxLayoutIDCMCStatistics.addWidget(self.beamSelector) self.setLayout(self.vBoxLayoutIDCMCStatistics) self.quantityButtonGroup.buttonClicked[int].connect( self.UpdateSelectedQuantity)
def __init__(self, name, prsr_section, hutol=None): syscfg = system_configuration.getInstance() self.name = name self.cache_dir = None self.dicom_match = dict() non_dicom_keys = list() if "density" in prsr_section and "composition" in prsr_section: # assuming this is a Schneider type HLUT logger.debug(f"{name} is a Schneider-type CT protocol") self.type = "Schneider" d = prsr_section["density"] c = prsr_section["composition"] self.density = os.path.join(syscfg["CT/density"], d) self.composition = os.path.join(syscfg["CT/composition"], c) if not os.path.exists(self.density): raise FileNotFoundError( "For Schneider protocol '{name}' the density file '{d}' cannot be found." ) if not os.path.exists(self.composition): raise FileNotFoundError( "For Schneider protocol '{name}' the composition file '{c}' cannot be found." ) self.hutol = syscfg[ 'hu density tolerance [g/cm3]'] if hutol is None else float( hutol) non_dicom_keys += ["density", "composition"] else: # assuming this is a commissioning type HLUT logger.debug( f"{name} is a CT protocol with direct HU-to-material tables (commissioning)" ) allowed_materials = syscfg['ct override list'].keys() self.type = "Commissioning" self.hu2mat_lines = list() self.density_lines = list() hulast = None for k in prsr_section.keys(): if not "," in k: continue try: hufromstr, hutillstr = k.split(",") hufrom = int(hufromstr) if hulast is not None: if hufrom != hulast: raise ValueError( f"HU intervals are not consecutive for {name}: gap between HU={hulast} and HU={HUfrom}" ) hutill = int(hutillstr) if not (hutill > hufrom): raise ValueError( f"wrong HU interval [{hufrom},{hutill}) for {name}: upper bound should be larger than lower bound" ) material = str(prsr_section[k]) if material not in allowed_materials: raise ValueError( f"material '{material}' not included in list of override materials in system configuration file '{syscfg['sysconfig']}'" ) self.hu2mat_lines.append(f"{hufrom} {hutill} {material}") density = syscfg['ct override list'][material] self.density_lines.append(f"{hufrom} {hutill} {density}") non_dicom_keys.append(k) except ValueError as ve: logger.error( f"something wrong with Hounsfield configuration {name}, please fix: '{ve}'" ) raise if len(self.density_lines) < 2: raise ValueError( f"CT protocol definition {self.name} incomplete/incorrect: need either a density & composition file (Schneider) or at least two HU-interval to material lines." ) for k, v in prsr_section.items(): if k in non_dicom_keys: continue dk = str(k).replace(" ", "") if dk in pydicom.datadict.keyword_dict.keys(): m = str(v).strip() if m == "": raise ValueError( "DICOM match criterion cannot be empty or only white space" ) logger.debug(f"Got DICOM match line: {k} = {m}") self.dicom_match[dk] = m else: # k is not a DICOM keyword, not a Schneider table nor a HU interval raise KeyError( f"I do not know what to do with this hlut.conf line: '{k} = {v}'" )
def PlanUpdate(self): #beams = self.details.rp_dataset.IonBeamSequence beams = self.details.bs_info.beams ctgeometry = self.details.run_with_CT_geometry logger.debug('updating {} beams'.format(len(beams))) self.beamTable.setRowCount(len(beams)) self.override_cache = dict() self.rs_combos = dict() self.rm_combos = dict() syscfg = system_configuration.getInstance() for i, beam in enumerate(beams): bml_name = beam.TreatmentMachineName bml = beamline_model.get_beamline_model_data( bml_name, syscfg['beamlines']) logger.debug('updating beam {}'.format(i)) if not beam.PrimaryDosimeterUnit == "NP": raise ValueError( "primary dosimetry unit should be 'NP', other units are not (yet) supported." ) #icps = beam.layers isoC = beam.IsoCenter if ctgeometry else self.details.PhantomISOinMM( beam.Name) j = self.headers.index("x | R-L [mm]") #self.beamTable.setItem(i,j,QtWidgets.QTableWidgetItem(str(isoC[0]))) self.rl_spinbox[beam.Name] = QtWidgets.QDoubleSpinBox() self.rl_spinbox[beam.Name].setRange(-1000., +1000.) self.rl_spinbox[beam.Name].setSingleStep(5.0) self.rl_spinbox[beam.Name].setValue(isoC[0]) self.rl_spinbox[beam.Name].setReadOnly(ctgeometry) self.beamTable.setCellWidget(i, j, self.rl_spinbox[beam.Name]) j = self.headers.index("y | A-P [mm]") #self.beamTable.setItem(i,j,QtWidgets.QTableWidgetItem(str(isoC[2]))) self.ap_spinbox[beam.Name] = QtWidgets.QDoubleSpinBox() self.ap_spinbox[beam.Name].setRange(-1000., +1000.) self.ap_spinbox[beam.Name].setSingleStep(5.0) self.ap_spinbox[beam.Name].setValue(isoC[2]) self.ap_spinbox[beam.Name].setReadOnly(ctgeometry) self.beamTable.setCellWidget(i, j, self.ap_spinbox[beam.Name]) j = self.headers.index("z | I-S [mm]") #self.beamTable.setItem(i,j,QtWidgets.QTableWidgetItem(str(isoC[1]))) self.is_spinbox[beam.Name] = QtWidgets.QDoubleSpinBox() self.is_spinbox[beam.Name].setRange(-1000., +1000.) self.is_spinbox[beam.Name].setSingleStep(5.0) self.is_spinbox[beam.Name].setValue(isoC[1]) self.is_spinbox[beam.Name].setReadOnly(ctgeometry) self.beamTable.setCellWidget(i, j, self.is_spinbox[beam.Name]) #j=self.headers.index("IsoCenter Name") #self.beamTable.setItem(i,j,QtWidgets.QTableWidgetItem("(TBD)")) j = self.headers.index("Patient Support Angle [deg]") self.beamTable.setItem( i, j, QtWidgets.QTableWidgetItem( str(beam.PatientSupportAngle) if ctgeometry else "NA")) #j=self.headers.index("Gap [mm]") #self.beamTable.setItem(i,j,QtWidgets.QTableWidgetItem("(TBD)")) j = self.headers.index("Beam Name") self.beamTable.setItem(i, j, QtWidgets.QTableWidgetItem(beam.Name)) j = self.headers.index("Radiation Type") self.beamTable.setItem( i, j, QtWidgets.QTableWidgetItem(beam.RadiationType)) j = self.headers.index("Treatment Machine") self.beamTable.setItem(i, j, QtWidgets.QTableWidgetItem(bml_name)) j = self.headers.index("Snout ID") self.beamTable.setItem(i, j, QtWidgets.QTableWidgetItem(beam.SnoutID)) j = self.headers.index("Snout Pos [mm]") logger.debug('halfway done with updating beam {}'.format(i)) self.beamTable.setItem( i, j, QtWidgets.QTableWidgetItem(str(beam.SnoutPosition))) j = self.headers.index("Gantry [deg]") self.beamTable.setItem( i, j, QtWidgets.QTableWidgetItem(str(beam.gantry_angle))) j = self.headers.index("Nr. of energy layers") self.beamTable.setItem( i, j, QtWidgets.QTableWidgetItem(str(beam.NumberOfEnergies))) j = self.headers.index("NP (1e6 / fx)") self.beamTable.setItem( i, j, QtWidgets.QTableWidgetItem( str(float(beam.FinalCumulativeMetersetWeight) / 1.0e6))) weights = [ float(w) / 1.0e6 for l in beam.layers for w in l.weights if w > 0 ] j = self.headers.index("Spot min") self.beamTable.setItem( i, j, QtWidgets.QTableWidgetItem(str(min(weights)))) j = self.headers.index("Spot max") self.beamTable.setItem( i, j, QtWidgets.QTableWidgetItem(str(max(weights)))) j = self.headers.index("Spot Tune ID") tunes = set([l.tuneID for l in beam.layers]) if len(tunes) > 1: logger.warn("{} spot tune IDs for beam {}".format( len(tunes), str(beam.BeamName))) self.beamTable.setItem(i, j, QtWidgets.QTableWidgetItem(tunes.pop())) #self.beamTable.setItem(i,j,QtWidgets.QTableWidgetItem("NO" if beam.NumberOfRangeShifters==0 else "YES")) logger.debug("adding RS combobox to TP table") rs_options, irs = label_combinations(bml.rs_labels, beam.RangeShifterIDs) self.rs_combos[beam.Name] = QtWidgets.QComboBox() self.rs_combos[beam.Name].addItems( ["NONE"] + ["+".join(opt) for opt in rs_options[1:]]) self.rs_combos[beam.Name].setCurrentIndex(irs) self.rs_combos[beam.Name].setEnabled(not ctgeometry) j = self.headers.index("Range Shifter") self.beamTable.setCellWidget(i, j, self.rs_combos[beam.Name]) logger.debug("adding RM combobox to TP table") rm_options, irm = label_combinations(bml.rm_labels, beam.RangeModulatorIDs) self.rm_combos[beam.Name] = QtWidgets.QComboBox() self.rm_combos[beam.Name].addItems( ["NONE"] + ["+".join(opt) for opt in rm_options[1:]]) self.rm_combos[beam.Name].setCurrentIndex(irm) self.rm_combos[beam.Name].setEnabled(not ctgeometry) j = self.headers.index("Range Modulators") self.beamTable.setCellWidget(i, j, self.rm_combos[beam.Name]) self.override_cache[beam.Name] = dict( rsa=rs_options, rsi=irs, rsc=self.rs_combos[beam.Name], rma=rm_options, rmi=irm, rmc=self.rm_combos[beam.Name]) for j in range(len(self.headers)): item = self.beamTable.item(i, j) if item: item.setFlags(QtCore.Qt.ItemIsEnabled) self.rs_combos[beam.Name].currentIndexChanged[int].connect( functools.partial(self.UpdateRSOverride, beam.Name)) self.rm_combos[beam.Name].currentIndexChanged[int].connect( functools.partial(self.UpdateRMOverride, beam.Name)) self.rl_spinbox[beam.Name].editingFinished.connect( functools.partial(self.UpdatePhantomIsoOverride, beam.Name)) self.ap_spinbox[beam.Name].editingFinished.connect( functools.partial(self.UpdatePhantomIsoOverride, beam.Name)) self.is_spinbox[beam.Name].editingFinished.connect( functools.partial(self.UpdatePhantomIsoOverride, beam.Name)) logger.debug('done with updating beam {}'.format(i))
def write_gate_macro_file(ct=True,**kwargs): """ This function writes the main macro for PSQA calculations for a specific beam in a beamset. beamset: name of the beamset uid: DICOM identifier of the full plan/beamset user: string that identifies the person who is responsible for this calculation spotfile: text file """ logger.debug("going to write mac files, got {} keywords".format(len(kwargs.keys()))) check(ct,**kwargs) logger.debug("basic checks went fine") syscfg = system_configuration.getInstance() beamline = kwargs["beamline"] kwargs["beamlinename"]=beamline.name kwargs["nmaxprimaries"]=2**31-1 # 2147483647, max number for a signed integer = max number of primaries that GATE can simulate radtype = kwargs["radtype"] isoC = kwargs["isoC"] # want Uncertainty? # wantU = (MCStatType.Xpct_unc_in_target == kwargs["mcstattype"]) #coreMinutes = (MCStatType.Nminutes_per_job == kwargs["mcstattype"]) #if coreMinutes: # kwargs["coreSeconds"] = 60*int(kwargs["minutes_per_job"]) #kwargs["wantU"] = str(wantU).lower() dose2w = True if ct: logger.debug("CT geometry for beamline {}".format(beamline.name)) kwargs["geometry"] = "CT" ct_bb = kwargs["ct_bb"] dose_center = kwargs["dose_center"] dose_size = kwargs["dose_size"] # in the dose actor, we need to give the position relative to the center of the CT image rot_box_size = 2.0001*np.max(np.abs(np.stack([ct_bb.mincorner-isoC,ct_bb.maxcorner-isoC])),axis=0) kwargs["xboxsize"] = float(rot_box_size[0]) kwargs["yboxsize"] = float(rot_box_size[1]) kwargs["zboxsize"] = float(rot_box_size[2]) # for CT we always request "dose to water" dose2w = True kwargs["wantDose"] = "false" kwargs["wantDose2W"] = "true" kwargs["label"]="CT-{beamset}-B{beamnr}-{beamname}-{beamlinename}".format(**kwargs) #kwargs["massmhd"]=os.path.basename(kwargs["ct_mhd"]).replace(".mhd","_mass.mhd") else: # for a water box we request normal dose to material # for other materials (e.g. PMMA) we request "dose to water" phantom = kwargs["phantom"] # this is an object of class phantom_specs, see system_configuration.py dose2w = phantom.dose_to_water # this is a boolean kwargs["wantDose2W"] = str(dose2w).lower() kwargs["wantDose"] = str(not dose2w).lower() kwargs["geometry"] = "PHANTOM" kwargs["phantom_macfile"] = os.path.join("data","phantoms",phantom.label+".mac") kwargs["phantom_name"] = phantom.label kwargs["phantom_move_x"] = -1.*isoC[0] kwargs["phantom_move_y"] = -1.*isoC[1] kwargs["phantom_move_z"] = -1.*isoC[2] kwargs["label"]="PHANTOM-{phantom_name}-{beamset}-B{beamnr}-{beamname}".format(**kwargs) logger.debug("PHANTOM geometry for beamline {}".format(beamline.name)) dosemhd="idc-{label}.mhd".format(**kwargs) dosedose="idc-{label}-".format(**kwargs) dosedose+="DoseToWater" if dose2w else "Dose" logger.debug("dose actor MHD output file name is {}".format(dosemhd)) kwargs["dosemhd"]=dosemhd kwargs["dosedosemhd"]=dosedose+".mhd" kwargs["dosedoseraw"]=dosedose+".raw" kwargs["stop_on_script_every_n_seconds"] = syscfg["stop on script actor time interval [s]"] dose_nvoxels=kwargs["dose_nvoxels"] kwargs["dnx"]=dose_nvoxels[0] kwargs["dny"]=dose_nvoxels[1] kwargs["dnz"]=dose_nvoxels[2] kwargs["materialsdb"]=syscfg["materials database"] kwargs["fulldate"]=time.strftime("%A %d %B ")+roman_year() kwargs["hhmmss"]=time.strftime("%H:%M:%S") kwargs["yyyymmdd"]=time.strftime("%Y-%m-%d") kwargs["user"]=syscfg["username"] kwargs["isocx"]=isoC[0] kwargs["isocy"]=isoC[1] kwargs["isocz"]=isoC[2] kwargs["srcprops"] = os.path.basename(beamline.source_properties_file(radtype)) if radtype.upper() != "PROTON": ion_details = radtype.split("_") if len(ion_details) != 4 or ion_details[0] != "ION": logger.error('Radiation type string should be either "PROTON" or of the form "ION_Z_A_Q"') raise RuntimeError("unsupported and/or ill-formatted radiation type {}".format(radtype)) kwargs["ionZ"]=ion_details[1] kwargs["ionA"]=ion_details[2] kwargs["ionQ"]=ion_details[3] ########## HEADER ########### header = """ ############################################################################### # Gate-based IDC for PSQA on CT or PHANTOM geometries # ############################################################################### # # Geometry = {geometry} # BeamSet = {beamset} # UID = {uid} # User = {user} # Beam Number = {beamnr} # Beam Name = {beamname} # Treatment Machine = {beamlinename} # Spot file = {spotfile} # Date (full) = {fulldate} # Date = {yyyymmdd} # Time = {hhmmss} # ############################################################################### /control/execute {{VISUMAC}} """.format(**kwargs) logger.debug("defined header section") ########## MATERIALS ########### materials_section = """ #===================================================== #= MATERIALS /gate/geometry/setMaterialDatabase data/{materialsdb} """.format(**kwargs) logger.debug("defined material section") ########## GEOMETRY ########### geometry_section = """ #===================================================== #= GEOMETRY #===================================================== # WORLD /gate/world/setMaterial Air /gate/world/geometry/setXLength 5.0 m /gate/world/geometry/setYLength 5.0 m /gate/world/geometry/setZLength 5.0 m # Make sure that G4_WATER is included in the geometry (and hence the dynamic database) # so that "DoseToWater" works. /gate/world/daughters/name water_droplet /gate/world/daughters/insert box /gate/water_droplet/setMaterial G4_WATER /gate/water_droplet/geometry/setXLength 1 mm /gate/water_droplet/geometry/setYLength 1 mm /gate/water_droplet/geometry/setZLength 1 mm /gate/water_droplet/placement/setTranslation -2499.5 -2499.5 -2499.5 mm /gate/water_droplet/vis/setColor cyan """ if beamline.beamline_details_mac_file: geometry_section += """ /control/execute mac/{} """.format(os.path.basename(beamline.beamline_details_mac_file)) logger.debug("defined beamline part of geometry section") for rs in kwargs["rsids"]: geometry_section += """ /control/execute mac/{} """.format(os.path.basename(beamline.rs_details_mac_file(rs))) logger.debug("added mac file for RS={}".format(rs)) for rm in kwargs["rmids"]: geometry_section += """ /control/execute mac/{} """.format(os.path.basename(beamline.rm_details_mac_file(rm))) logger.debug("added mac file for rifi={}".format(rm)) if ct: # TODO: correct rotation axis and order for gantry & patient angles geometry_section += """ # Patient virtual container: for couch rotation /gate/world/daughters/name patient_box /gate/world/daughters/insert box /gate/patient_box/setMaterial Air /gate/patient_box/geometry/setXLength {xboxsize} mm /gate/patient_box/geometry/setYLength {yboxsize} mm /gate/patient_box/geometry/setZLength {zboxsize} mm /gate/patient_box/vis/setColor yellow # CT Volume /gate/patient_box/daughters/name patient /gate/patient_box/daughters/insert ImageNestedParametrisedVolume /gate/geometry/setMaterialDatabase {HUmaterials} /gate/patient/geometry/setHUToMaterialFile {HU2mat} /gate/patient/geometry/setImage {ct_mhd} /gate/patient/geometry/TranslateTheImageAtThisIsoCenter {isocx} {isocy} {isocz} mm /gate/patient_box/placement/setRotationAxis 0 1 0 /gate/patient_box/placement/setRotationAngle {mod_patient_angle} deg """.format(**kwargs) logger.debug("added HU material generator and voxelized image for CT") else: geometry_section += """ /control/alias {phantom_name}_move_x {phantom_move_x} /control/alias {phantom_name}_move_y {phantom_move_y} /control/alias {phantom_name}_move_z {phantom_move_z} /control/alias phantom_name {phantom_name} /control/execute {phantom_macfile} """.format(**kwargs) ########## PHYSICS ########### physics_section = """ #===================================================== #= PHYSICS /gate/physics/addPhysicsList {physicslist} #===================================================== #= GATE BASIC PARAMETERS """.format(**kwargs) logger.debug("physics = {}".format(kwargs["physicslist"])) if ct: physics_section += """ /control/execute data/CT/ct-parameters.mac """ else: physics_section += """ /control/execute data/phantoms/phantom-parameters.mac """ ########## OUTPUT ########### output_section = """ #===================================================== #= OUTPUTS #===================================================== #===================================================== # Statistics actor #===================================================== /gate/actor/addActor SimulationStatisticActor stat /gate/actor/stat/saveEveryNSeconds 120 /gate/actor/stat/save {{OUTPUTDIR}}/statActor-{label}.txt #===================================================== # Dose Actor #===================================================== /gate/actor/addActor DoseActor dose /gate/actor/dose/save {{OUTPUTDIR}}/{dosemhd} """.format(**kwargs) if ct: output_section += """ /gate/actor/dose/attachTo patient """.format(**kwargs) else: output_section += """ /gate/actor/dose/attachTo {phantom_name} """.format(**kwargs) output_section += """ /gate/actor/dose/setResolution {dnx} {dny} {dnz} /gate/actor/dose/stepHitType random /gate/actor/dose/enableEdep false /gate/actor/dose/enableUncertaintyEdep false /gate/actor/dose/enableDose true /gate/actor/dose/enableDoseToWater {wantDose2W} """.format(**kwargs) logger.debug("defined output section") noop_filepath="mac/stop_on_script_{label}_noop.mac".format(**kwargs) with open(noop_filepath,"w") as noop: noop.write("""/control/echo NOT adding any stop-on-script actor (QT test run)\n""") stop_on_script_filepath="mac/stop_on_script_{label}.mac".format(**kwargs) with open(stop_on_script_filepath,"w") as sosmac: sosmac.write( """ /control/echo "ADDING stop on script actor" /gate/actor/addActor StopOnScriptActor stopWhenReady /gate/actor/stopWhenReady/saveEveryNSeconds {stop_on_script_every_n_seconds} /gate/actor/stopWhenReady/saveAllActors true /gate/actor/stopWhenReady/save mac/check_the_flags_{label}.sh """.format(**kwargs)) with open("mac/check_the_flags_{label}.sh".format(**kwargs),"w") as chksh: chksh.write(""" workdir=$(dirname $(realpath tmp)) echo "workdir is $workdir" stopflag="STOP_{dosedosemhd}" if test -r "$workdir/$stopflag" ; then echo "found STOP flag $stopflag, exit with return value 1 to stop simulation" exit 1 fi echo "did not find STOP flag $stopflag, returning 0 to continue simulation" d=output.$clusterid.$procid ls -ld $d ls -lrt $d ls -ld tmp/$d ls -lrt tmp/$d ls "$d/{dosedosemhd}" "$d/{dosedoseraw}" "$d/statActor-{label}.txt" type locked_copy.py locked_copy.py -v -d tmp/$d/ -l tmp/$d/{dosedosemhd}.lock "$d/{dosedosemhd}" "$d/{dosedoseraw}" "$d/statActor-{label}.txt" exit 0 """.format(**kwargs)) output_section += """ /control/strif {{RUNMAC}} == mac/run_all.mac {} /control/strif {{RUNMAC}} == mac/run_qt.mac {} """.format(stop_on_script_filepath,noop_filepath) # elif coreMinutes: # Nseconds_filepath="mac/stop_after_N_seconds_{label}.mac".format(**kwargs) # with open(Nseconds_filepath,"w") as Nsec_mac: # Nsec_mac.write( """ #/control/echo adding stop on script actor for runtime threshold #/gate/actor/addActor StopOnScriptActor stopAfterNSeconds #/gate/actor/stopAfterNSeconds/saveEveryNSeconds {coreSeconds} #/gate/actor/stopAfterNSeconds/saveAllActors true #/gate/actor/stopAfterNSeconds/save mac/stop_after_N_seconds_{label}.sh #""".format(**kwargs)) # with open("mac/stop_after_N_seconds_{label}.sh".format(**kwargs),"w") as chksh: # chksh.write(""" #echo "clusterid=$clusterid" #echo "procid=$procid" #echo "time is up!" #date #condor_q $clusterid.$procid -run -nobatch -allusers -global #ddhhmmss=$(condor_q $clusterid.$procid -run -nobatch -allusers -global | tail -1 | awk '{{print $5}}') #dd=$(echo $ddhhmmss | cut -f1 -d+ ) #hhmmss=$(echo $ddhhmmss | cut -f2 -d+ ) #hh=$(echo $hhmmss | cut -f1 -d: ) #mm=$(echo $hhmmss | cut -f2 -d: ) #ss=$(echo $hhmmss | cut -f3 -d: ) #hh=$[$hh+24*$dd] #mm=$[$mm + 60*$hh] #ss=$[$ss + 60*$mm + 30] #mm2=$[$ss / 60] #echo ran for mm=$mm mm2=$mm2 minutes #exit 1 #echo this message will never be printed #""".format(**kwargs)) # output_section += """ #/control/strif {{RUNMAC}} == mac/run_all.mac {} #/control/strif {{RUNMAC}} == mac/run_qt.mac {} #""".format(Nseconds_filepath,noop_filepath) ########## INIT ########### init_section = """ #===================================================== #= INITIALIZATION #===================================================== /gate/run/initialize /gate/physics/print {{OUTPUTDIR}}/PHYSICS.txt """.format(**kwargs) ########## SOURCE ########### source_section = """ #===================================================== #= SOURCE #===================================================== /gate/source/addSource PBS TPSPencilBeam /gate/source/PBS/setTestFlag false """ if radtype.lower() == "proton": logger.debug("configuring protons") source_section += """ /gate/source/PBS/setParticleType proton """.format(**kwargs) else: logger.debug("configuring ions (probably carbons)") source_section += """ /gate/source/PBS/setParticleType GenericIon /gate/source/PBS/setIonProperties {ionZ} {ionA} {ionQ} """.format(**kwargs) source_section += """ /gate/source/PBS/setPlan {spotfile} # One beam at the time... /gate/source/PBS/setAllowedFieldID {beamnr} /gate/source/PBS/setSourceDescriptionFile data/{srcprops} /gate/source/PBS/setSpotIntensityAsNbIons true /gate/source/PBS/setBeamConvergenceXTheta {{{radtype}_CONVERGENCE_XTHETA}} /gate/source/PBS/setBeamConvergenceYPhi {{{radtype}_CONVERGENCE_YPHI}} """.format(**kwargs) logger.debug("defined source section") ########## RUN ########### run_section = """ #===================================================== #= RUN #===================================================== /gate/random/setEngineName MersenneTwister /control/execute {{RUNMAC}} /gate/application/start """.format(**kwargs) logger.debug("defined run section") ########## ALIAS ########### #main_fname=os.path.join("mac","main-{geometry}-{beamset}-{beamname}-{beamnr}-{beamlinename}.mac".format(**kwargs)) main_fname=os.path.join("mac",dosemhd.replace(".mhd",".mac")) f=open(main_fname,"w") f.write(header) f.write(materials_section) f.write(geometry_section) f.write(physics_section) f.write(output_section) f.write(init_section) f.write(source_section) f.write(run_section) f.close() logger.debug("wrote main mac file: {}".format(main_fname)) fname=os.path.join("mac","run_all.mac") f=open(fname,"w") f.write(""" /gate/random/setEngineSeed {{RNGSEED}} /gate/application/setTotalNumberOfPrimaries {nmaxprimaries} """.format(**kwargs)) f.close() fname=os.path.join("mac","run_qt.mac") f=open(fname,"w") f.write(""" /gate/random/setEngineSeed 42 /gate/application/setTotalNumberOfPrimaries 100 """) f.close() fname=os.path.join("mac","visu.mac") f=open(fname,"w") f.write(""" /vis/enable /vis/open OGLIQt /vis/drawVolume /vis/viewer/flush /vis/scene/add/trajectories /vis/scene/endOfEventAction accumulate /vis/scene/add/axes 0 0 0 50 mm /vis/scene/add/text 1 0 0 cm 20 0 0 X /vis/scene/add/text 0 1 0 cm 20 0 0 Y /vis/scene/add/text 0 0 1 cm 20 0 0 Z /vis/viewer/set/viewpointThetaPhi 70 30 /vis/viewer/zoom 5 """) f.close() fname=os.path.join("mac","novisu.mac") f=open(fname,"w") f.write(""" /vis/disable """) f.close() logger.debug("wrote auxiliary mac files") logger.debug("DONE: wrote all mac files") return main_fname,dosemhd
def _populate_RUNGATE_submit_directory(self): """ Create the content of the Gate directory: how to run the simulation. TODO: This method implementation is very long, should be broken up in smaller entities. TODO: In particular, the HTCondor-specific stuff should be separated out, so that it will be easier to also support other cluster job management systems, like SLURM and OpenPBS. """ #################### syscfg = system_configuration.getInstance() #################### save_cwd = os.getcwd() assert (os.path.isdir(self._RUNGATE_submit_directory)) os.chdir(self._RUNGATE_submit_directory) for d in ["output", "mac", "data", "logs"]: os.mkdir(d) #################### shutil.copy( os.path.join(syscfg['commissioning'], syscfg['materials database']), os.path.join("data", syscfg['materials database'])) ct = self.details.run_with_CT_geometry ct_bb = None if ct: #shutil.copytree(syscfg["CT"],os.path.join("data","CT")) dataCT = os.path.join(os.path.realpath("./data"), "CT") os.mkdir(dataCT) shutil.copy(os.path.join(syscfg["CT"], "ct-parameters.mac"), os.path.join(dataCT, "ct-parameters.mac")) #### HUtol=str(syscfg['hu density tolerance [g/cm3]']) #### hlutdensity=os.path.realpath(self.details.HLUTdensity) #### hlutmaterials=os.path.realpath(self.details.HLUTmaterials) #### # the name of the cache directory contains the MD5 sum of the contents of the density & material files as well as the HUtol value #### cache_dir = hlut_cache_dir(density = hlutdensity, #### composition = hlutmaterials, #### HUtol = HUtol, #### create = False) # returns None if cache directory does not exist #### if cache_dir is None: #### # generate cache #### # TODO: IDC also does this (in UpdateHURange), should we really do this also here? #### logger.info(f"going to generate missing material cache dir {cache_dir}") #### success, cache_dir = generate_hlut_cache(hlutdensity,hlutmaterials,HUtol) #### if not success: #### raise RuntimeError("failed to create material cache for HLUT={hlutdensity} and composition={hlutmaterials}") #### cached_hu2mat=os.path.join(cache_dir,"patient-HU2mat.txt") #### cached_humdb=os.path.join(cache_dir,"patient-HUmaterials.db") all_hluts = hlut_conf.getInstance() # TODO: should 'idc_details' ask the user about a HU density tolerance value? # TODO: should we try to catch the exceptions that 'all_hluts' might throw at us? cached_hu2mat_txt, cached_humat_db = all_hluts[ self.details.ctprotocol_name].get_hu2mat_files() hudensity = all_hluts[ self.details.ctprotocol_name].get_density_file() hu2mat_txt = os.path.join(dataCT, os.path.basename(cached_hu2mat_txt)) humat_db = os.path.join(dataCT, os.path.basename(cached_humat_db)) shutil.copy(cached_hu2mat_txt, hu2mat_txt) shutil.copy(cached_humat_db, humat_db) mcpatientCT_filepath = os.path.join( dataCT, self.details.uid.replace(".", "_") + ".mhd") ct_bb, ct_nvoxels = self.details.WritePreProcessingConfigFile( self._RUNGATE_submit_directory, mcpatientCT_filepath, hu2mat_txt, hudensity) msg = "IDC with CT geometry" else: # TODO: should we try to only copy the relevant phantom data, instead of the entire phantom collection? shutil.copytree(syscfg["phantoms"], os.path.join("data", "phantoms")) msg = "IDC with PHANTOM geometry" logger.debug(msg) self._summary += msg + '\n' #################### beamset = self.details.bs_info beamsetname = re.sub(self._badchars, "_", beamset.name) if ct: # the name has to end in PLAN plan_dose_file = f"idc-CT-{beamsetname}-PLAN" else: phantom_name = self.details.PhantomSpecs.label plan_dose_file = f"idc-PHANTOM-{phantom_name}-{beamsetname}-PLAN" spotfile = os.path.join( "data", "TreatmentPlan4Gate-{}.txt".format(beamset.name.replace(" ", "_"))) gate_plan = gate_pbs_plan_file(spotfile, allow0=True) gate_plan.import_from(beamset) ncores = self._get_ncores() beamlines = list() for beam in beamset.beams: logger.debug(f"configuring beam {beam.Name}") bmlname = beam.TreatmentMachineName if self.details.beamline_override is None else self.details.beamline_override logger.debug(f"beamline name is {bmlname}") beamnr = beam.Number beamname = re.sub(self._badchars, "_", beam.Name) if beamname == beam.Name: self._summary += "beam: '{}'\n".format(beamname) else: self._summary += "beam: '{}'/'{}'\n".format( beam.Name, beamname) radtype = beam.RadiationType if radtype.upper() == 'PROTON': physlist = syscfg['proton physics list'] elif radtype.upper()[:3] == 'ION': physlist = syscfg['ion physics list'] else: raise RuntimeError( "don't know which physics list to use for radiation type '{}'" .format(radtype)) if not self.details.BeamIsSelected(beam.Name): msg = "skip simulation for de-selected beam name={} nr={} machine={}.".format( beamname, beamnr, bmlname) logger.warn(msg) continue # TODO: do we need this distinction between ncores and njobs? # maybe we'll need this for when the uncertainty goal needs to apply to the plan dose instead of beam dose? njobs = ncores if self.details.run_with_CT_geometry: rsids = beam.RangeShifterIDs rmids = beam.RangeModulatorIDs else: rsids = self.details.RSOverrides.get(beam.Name, beam.RangeShifterIDs) rmids = self.details.RMOverrides.get(beam.Name, beam.RangeModulatorIDs) rsflag = "(as PLANNED)" if rsids == beam.RangeShifterIDs else "(OVERRIDE)" rmflag = "(as PLANNED)" if rmids == beam.RangeModulatorIDs else "(OVERRIDE)" bml = beamline_model.get_beamline_model_data( bmlname, syscfg['beamlines']) if not bml.has_radtype(radtype): msg = "BeamNumber={}, BeamName={}, BeamLine={}\n".format( beamnr, beamname, bmlname) msg += "* ERROR: simulation not possible: no source props file for radiation type {}\n".format( radtype) logger.warn(msg) self._summary += msg continue msg = "BeamNumber={}, BeamName={}, BeamLine={}\n".format( beamnr, beamname, bmlname) msg += "* Range shifter(s): {} {}\n".format( ("NONE" if len(rsids) == 0 else ",".join(rsids)), rsflag) msg += "* Range modulator(s): {} {}\n".format( ("NONE" if len(rmids) == 0 else ",".join(rmids)), rmflag) #msg += "* {} primaries per job => est. {} s/job.\n".format(nprim,dt) logger.debug(msg) if rsflag == "(OVERRIDE)" or rmflag == "(OVERRIDE)": self._summary += msg if self.details.dosegrid_changed: self._summary += "dose grid resolution changed to {}\n".format( self.details.GetNVoxels()) #TODO: change api for 'write_gate_macro_file' to take fewer arguments macfile_input = dict(beamset=beamsetname, uid=self.details.uid, spotfile=spotfile, beamline=bml, beamnr=beamnr, beamname=beamname, radtype=radtype, rsids=rsids, rmids=rmids, physicslist=physlist) if ct: nominal_patient_angle = beam.patient_angle mod_patient_angle = (360.0 - beam.patient_angle) % 360.0 macfile_input.update(ct=True, ct_mhd=mcpatientCT_filepath, dose_center=self.details.GetDoseCenter(), dose_size=self.details.GetDoseSize(), ct_bb=ct_bb, dose_nvoxels=ct_nvoxels, mod_patient_angle=mod_patient_angle, gantry_angle=beam.gantry_angle, isoC=np.array(beam.IsoCenter), HU2mat=hu2mat_txt, HUmaterials=humat_db) else: macfile_input.update( ct=False, dose_nvoxels=self.details.GetNVoxels(), isoC=np.array(self.details.PhantomISOinMM(beam.Name)), phantom=self.details.PhantomSpecs) # the following two lines are not strictly necessary phpath = self.details.PhantomSpecs.mac_file_path shutil.copy( phpath, os.path.join("data", "phantoms", os.path.basename(phpath))) main_macfile, beam_dose_mhd = write_gate_macro_file( **macfile_input) assert (main_macfile not in self._mac_files) # should never happen self._mac_files.append(main_macfile) # def_dose_corr_factor = syscfg['(tmp) correction factors'][ "default"] dose_corr_key = (bmlname + "_" + radtype).lower() dose_corr_factor = syscfg['(tmp) correction factors'].get( dose_corr_key, def_dose_corr_factor) # self._qspecs[beamname] = dict( nJobs=str(njobs), #nMC=str(nprim), #nMCtot=str(nprimtot), origname=beam.Name, dosecorrfactor=str(dose_corr_factor), dosemhd=beam_dose_mhd, macfile=main_macfile, dose2water=str(ct or self.details.PhantomSpecs.dose_to_water), isocenter=" ".join(["{}".format(v) for v in beam.IsoCenter])) shutil.copy(bml.source_properties_file(radtype), "data") for rs in rsids: dest = os.path.join( "mac", os.path.basename(bml.rs_details_mac_file(rs))) if not os.path.exists(dest): shutil.copy(bml.rs_details_mac_file(rs), dest) for rm in rmids: dest = os.path.join( "mac", os.path.basename(bml.rm_details_mac_file(rm))) if not os.path.exists(dest): shutil.copy(bml.rm_details_mac_file(rm), dest) if bmlname in beamlines: continue if bml.beamline_details_mac_file: shutil.copy(bml.beamline_details_mac_file, "mac") for a in bml.beamline_details_aux: dest = os.path.join("data", os.path.basename(a)) if os.path.exists(dest): raise RuntimeError("CONFIG ERROR") if os.path.isdir(a): shutil.copytree(a, dest) else: shutil.copy(a, dest) for a in bml.common_aux: dest = os.path.join("data", os.path.basename(a)) if os.path.exists(dest): continue if os.path.isdir(a): shutil.copytree(a, dest) else: shutil.copy(a, dest) beamlines.append(bmlname) logger.debug("copied all beam line models into data directory") logger.debug("wrote mac files for all beams to be simulated") #self._summary + "{} seconds estimated for simulation of whole plan".format(self._ect) rsd = self._RUNGATE_submit_directory os.makedirs(os.path.join(rsd, "tmp"), exist_ok=True ) # the 'mode' argument is ignored (not only on Windows) os.chmod(os.path.join(rsd, "tmp"), mode=0o777) with open("RunGATE.sh", "w") as jobsh: jobsh.write("#!/bin/bash\n") jobsh.write("set -x\n") jobsh.write("set -e\n") jobsh.write("whoami\n") jobsh.write("who am i\n") jobsh.write("date\n") jobsh.write("echo $# arguments\n") jobsh.write('echo "pwd=$(pwd)"\n') jobsh.write("macfile=$1\n") jobsh.write("export clusterid=$2\n") jobsh.write("export procid=$3\n") jobsh.write("pwd -P\n") jobsh.write("pwd -L\n") jobsh.write(f"cd {rsd}\n") jobsh.write("pwd -P\n") jobsh.write("pwd -L\n") #jobsh.write("tar zxvf macdata.tar.gz\n") #if ct: # #jobsh.write("cat data/HUoverrides.txt >> data/patient-HU2mat.txt\n") # jobsh.write("tar zxvf ct.tar.gz\n") jobsh.write("outputdir=./output.$clusterid.$procid\n") jobsh.write("tmpoutputdir=./tmp/output.$clusterid.$procid\n") jobsh.write("mkdir $outputdir\n") jobsh.write("mkdir $tmpoutputdir\n") jobsh.write("chmod 777 ./tmp/$outputdir\n") #jobsh.write("mkdir {}/$outputdir\n".format(rsd)) #jobsh.write('touch "$outputdir/START_$(basename $macfile)"\n') #jobsh.write("ln -s {}/$outputdir\n".format(rsd)) #jobsh.write("ln -s {}/mac\n".format(rsd)) #jobsh.write("ln -s {}/data\n".format(rsd)) jobsh.write("source {}\n".format( os.path.join(syscfg['bindir'], "IDEAL_env.sh"))) jobsh.write("source {}\n".format(syscfg['gate_env.sh'])) jobsh.write("seed=$[1000*clusterid+procid]\n") jobsh.write("echo rng seed is $seed\n") jobsh.write("ret=0\n") # with the following construction, Gate can crash without DAGman removing all remaining jobs in the queue jobsh.write( "Gate -a[RNGSEED,$seed][RUNMAC,mac/run_all.mac][VISUMAC,mac/novisu.mac][OUTPUTDIR,$outputdir] $macfile && echo GATE SUCCEEDED || ret=$? \n" ) jobsh.write( "if [ $ret -ne 0 ] ; then echo GATE FAILED WITH EXIT CODE $ret; fi\n" ) # the following is used both in postprocessing and by the job_control_daemon jobsh.write("echo $ret > $outputdir/gate_exit_value.txt\n") jobsh.write("du -hcs *\n") jobsh.write("date\n") jobsh.write("echo SECONDS=$SECONDS\n") jobsh.write("exit 0\n") os.chmod("RunGATE.sh", stat.S_IREAD | stat.S_IRWXU) logger.debug("wrote run shell script") with open("RunGATEqt.sh", "w") as jobsh: jobsh.write("#!/bin/bash\n") jobsh.write("set -x\n") jobsh.write("set -e\n") jobsh.write("macfile=$1\n") jobsh.write("outputdir=output_qt\n") jobsh.write("rm -rf $outputdir\n") jobsh.write("mkdir -p $outputdir\n") jobsh.write("source {}\n".format(syscfg['gate_env.sh'])) if ct: #jobsh.write("cat data/HUoverrides.txt >> data/patient-HU2mat.txt\n") jobsh.write( "echo running preprocess, may take a minute or two...\n") jobsh.write("time {}/preprocess_ct_image.py\n".format( syscfg["bindir"])) jobsh.write("echo starting Gate, may take a minute...\n") jobsh.write( "Gate --qt -a[RUNMAC,mac/run_qt.mac][VISUMAC,mac/visu.mac][OUTPUTDIR,$outputdir] $macfile\n" ) jobsh.write("du -hcs *\n") os.chmod("RunGATEqt.sh", stat.S_IREAD | stat.S_IRWXU) logger.debug("wrote run debugging shell script with GUI") # TODO: write the condor stuff directly in python? input_files = [ "RunGATE.sh", "macdata.tar.gz", "{}/locked_copy.py".format(syscfg["bindir"]) ] if ct: input_files.append("ct.tar.gz") with open("RunGATE.submit", "w") as jobsubmit: jobsubmit.write("universe = vanilla\n") jobsubmit.write("executable = {}/RunGATE.sh\n".format( self._RUNGATE_submit_directory)) jobsubmit.write("should_transfer_files = NO\n") jobsubmit.write(f'+workdir = "{self._RUNGATE_submit_directory}"\n') jobsubmit.write("priority = {}\n".format(self.details.priority)) jobsubmit.write("request_cpus = 1\n") # cluster job diagnostics: jobsubmit.write("output = logs/stdout.$(CLUSTER).$(PROCESS).txt\n") jobsubmit.write("error = logs/stderr.$(CLUSTER).$(PROCESS).txt\n") jobsubmit.write("log = logs/stdlog.$(CLUSTER).$(PROCESS).txt\n") # boiler plate jobsubmit.write("RunAsOwner = true\n") jobsubmit.write("nice_user = false\n") jobsubmit.write("next_job_start_delay = {}\n".format( syscfg["htcondor next job start delay [s]"])) jobsubmit.write("notification = error\n") # the actual submit command: for beamname, qspec in self._qspecs.items(): origname = qspec["origname"] jobsubmit.write("request_memory = {}\n".format( self.details.calculate_ram_request_mb(origname))) jobsubmit.write( "arguments = {} $(CLUSTER) $(PROCESS)\n".format( qspec['macfile'])) jobsubmit.write("queue {}\n".format(qspec['nJobs'])) os.chmod("RunGATE.submit", stat.S_IREAD | stat.S_IWUSR) logger.debug("wrote condor submit file") self.details.WritePostProcessingConfigFile( self._RUNGATE_submit_directory, self._qspecs, plan_dose_file) with open("RunGATE.dagman", "w") as dagman: if ct: dagman.write( "SCRIPT PRE rungate {}/preprocess_ct_image.py\n".format( syscfg["bindir"])) dagman.write("JOB rungate ./RunGATE.submit\n") dagman.write( "SCRIPT POST rungate {}/postprocess_dose_results.py\n".format( syscfg["bindir"])) os.chmod("RunGATE.dagman", stat.S_IREAD | stat.S_IWUSR) logger.debug("wrote condor dagman file") with tarfile.open("macdata.tar.gz", "w:gz") as tar: tar.add("mac") tar.add("data") logger.debug("wrote gzipped tar file with 'data' and 'mac' directory") os.chdir(save_cwd)