def recalculate_plan_complexities_from_beams(): with DVH_SQL() as cnx: uids = cnx.get_unique_values('Plans', 'study_instance_uid') for uid in uids: try: condition = "study_instance_uid = '%s'" % uid beam_complexities = cnx.query( 'Beams', 'fx_count, complexity, fx_grp_number', condition) complexity = {} fx_counts = {} for row in beam_complexities: fx_count, beam_complexity, fx_group_number = tuple(row) if fx_group_number not in complexity: complexity[fx_group_number] = 0.0 fx_counts[fx_group_number] = fx_count complexity[fx_group_number] += beam_complexity total_fx = float(sum([fx for fx in fx_counts.values()])) plan_complexity = sum([ c * fx_counts[fx_grp] for fx_grp, c in complexity.items() ]) / total_fx except Exception as e: msg = "tools.utilities.recalculate_plan_complexities_from_beams: failed on uid = %s" % uid push_to_log(e, msg=msg) plan_complexity = None if plan_complexity is not None: cnx.update('Plans', 'complexity', plan_complexity, condition)
def echo_sql_db(config=None, db_type='pgsql', group=1): """ Echo the database using stored or provided credentials :param config: database login credentials :type config: dict :param db_type: either 'pgsql' or 'sqlite' :type db_type: str :param group: either group 1 or 2 :type group: int :return: True if connection could be established :rtype: bool """ try: if config: if db_type == 'pgsql' and ('dbname' not in list(config) or 'port' not in list(config)): return False cnx = DVH_SQL(config, db_type=db_type, group=group) else: cnx = DVH_SQL(group=group) cnx.close() return True except Exception as e: if type(e) not in [psycopg2.OperationalError, sqlite3.OperationalError]: push_to_log(e, msg='Unknown Error during SQL Echo') return False
def delete_file(file_path): try: if isfile(file_path): unlink(file_path) elif isdir(file_path): shutil.rmtree(file_path) except Exception as e: push_to_log(e, msg='tools.utilities.delete_file: %s' % file_path)
def get_physician_from_uid(uid): with DVH_SQL() as cnx: results = cnx.query('Plans', 'physician', "study_instance_uid = '" + uid + "'") if len(results) > 1: msg = 'roi_name_manager.get_physician_from_uid: multiple plans with this study_instance_uid exist: %s' % uid push_to_log(msg=msg) return str(results[0][0])
def is_options_file_valid(self): try: current_checksum = self.calculate_checksum() stored_checksum = self.load_stored_checksum() if current_checksum == stored_checksum: return True except Exception as e: msg = 'Options.is_options_file_valid: Corrupted options file detected. Loading default options.' push_to_log(e, msg=msg) return False
def sync_spin_buttons(self): if self.x_axis: index = self.choices.index(self.x_axis) self.spin_button_x_axis.SetValue(len(self.choices) - 1 - index) index = self.choices.index(self.y_axis) self.spin_button_y_axis.SetValue(len(self.choices) - 1 - index) else: msg = 'RegressionFrame.sync_spin_buttons: x-axis choice is empty.' push_to_log(msg=msg)
def edit_study_uid(abs_file_path, study_uid): """ Change the StudyInstanceUID of a DICOM file :param abs_file_path: absolute file path of the DICOM file :param study_uid: new StudyInstanceUID """ try: ds = pydicom.read_file(abs_file_path, force=True) ds.StudyInstanceUID = study_uid ds.save_as(abs_file_path) except Exception as e: push_to_log(e, abs_file_path)
def set_option(self, attr, value): """ Change or create an option value :param attr: name of option :type attr: str :param value: value of option """ if not hasattr(self, attr): msg = 'Options.set_option: %s did not previously exist' % attr push_to_log(msg=msg) setattr(self, attr, value) self.is_edited = True
def load(self): self.is_edited = False if isfile(OPTIONS_PATH) and self.is_options_file_valid: try: with open(OPTIONS_PATH, 'rb') as infile: loaded_options = pickle.load(infile) self.upgrade_options(loaded_options) except Exception as e: msg = 'Options.load: Options file corrupted. Loading default options.' push_to_log(e, msg=msg) loaded_options = {} for key, value in loaded_options.items(): if hasattr(self, key): setattr(self, key, value)
def load_roi_types_from_file(self, physician): if self.is_physician(physician): file_path = os.path.join(PREF_DIR, 'physician_%s.rtype' % physician) if os.path.isfile(file_path): with open(file_path, 'r') as doc: for line in doc: if not line: continue if line.count(':') == 1: try: physician_roi, roi_type = tuple(line.split(':')) self.physicians[physician].rois[physician_roi.strip()].roi_type = roi_type.strip() except Exception as e: msg = 'DatabaseROIs.load_roi_types_from_file: ' \ 'Could not import %s roi_type for physician %s' % (physician_roi, physician) push_to_log(e, msg=msg)
def delete_column(self, column): """ Delete the specified column data and the layout :param column: column to be deleted :type column: str """ if column in self.keys: index = self.columns.index(column) if self.layout: try: self.layout.DeleteColumn(index) except Exception as e: msg = 'DataTable.delete_column: Could not delete column in layout' push_to_log(e, msg=msg) self.data.pop(column) self.columns.pop(index)
def beam_complexity(cnx, study_instance_uid): """ :param cnx: connection to DVHA SQL database :type cnx: DVH_SQL :param study_instance_uid: study_instance_uid in SQL database :type study_instance_uid: str """ rt_plan_query = cnx.query('DICOM_Files', 'folder_path, plan_file', "study_instance_uid = '%s'" % study_instance_uid)[0] rt_plan_file_path = join_path(rt_plan_query[0], rt_plan_query[1]) rt_plan = dicom.read_file(rt_plan_file_path) for beam_num, beam in enumerate(rt_plan.BeamSequence): try: condition = "study_instance_uid = '%s' and beam_number = '%s'" % ( study_instance_uid, (beam_num + 1)) meterset = float(cnx.query('Beams', 'beam_mu', condition)[0][0]) mlca_data = BeamAnalyzer(beam, meterset, ignore_zero_mu_cp=True) mlc_keys = ['area', 'x_perim', 'y_perim', 'cmp_score', 'cp_mu'] summary_stats = { key: calc_stats(mlca_data.summary[key]) for key in mlc_keys } column_vars = { 'area': 'area', 'x_perim': 'x_perim', 'y_perim': 'y_perim', 'complexity': 'cmp_score', 'cp_mu': 'cp_mu' } stat_map = {'min': 5, 'mean': 3, 'median': 2, 'max': 0} for c in list(column_vars): for s in list(stat_map): value = summary_stats[column_vars[c]][stat_map[s]] column = "%s_%s" % (c, s) cnx.update('Beams', column, value, condition) cnx.update('Beams', 'complexity', np.sum(mlca_data.summary['cmp_score']), condition) except Exception as e: msg = 'db.update.beam_complexity: MLC Analyzer fail for beam number %s and uid %s' % \ ((beam_num+1), study_instance_uid) push_to_log(e, msg=msg)
def update(self, table_name, column, value, condition_str): """ Change the data in the database. :param table_name: 'DVHs', 'Plans', 'Rxs', 'Beams', or 'DICOM_Files' :type table_name: str :param column: SQL column to be updated :type column: str :param value: value to be set :type value: str or float or int :param condition_str: a condition in SQL syntax :type condition_str: str """ try: float(value) value_is_numeric = True except ValueError: value_is_numeric = False if '::date' in str(value): amend_type = [ '', '::date' ][self.db_type == 'pgsql'] # sqlite3 does not support ::date value = "'%s'%s" % ( value.strip('::date'), amend_type ) # augment value for postgresql date formatting elif value_is_numeric: value = str(value) elif 'null' == str(value.lower()): value = "NULL" else: value = "'%s'" % str(value) # need quotes to input a string update = "Update %s SET %s = %s WHERE %s" % (table_name, column, value, condition_str) try: self.cursor.execute(update) self.cnx.commit() except Exception as e: push_to_log(e, msg="Database update failure!")
def plan_complexity(cnx, study_instance_uid): """ :param cnx: connection to DVHA SQL database :type cnx: DVH_SQL :param study_instance_uid: study_instance_uid in SQL database :type study_instance_uid: str """ condition = "study_instance_uid = '%s'" % study_instance_uid beam_data = query('Beams', 'complexity, beam_mu', condition) scores = [row[0] for row in beam_data] include = [i for i, score in enumerate(scores) if score] scores = [score for i, score in enumerate(scores) if i in include] beam_mu = [row[1] for i, row in enumerate(beam_data) if i in include] plan_mu = np.sum(beam_mu) if plan_mu: complexity = np.sum(np.multiply(scores, beam_mu)) / plan_mu cnx.update('Plans', 'complexity', complexity, "study_instance_uid = '%s'" % study_instance_uid) else: msg = 'db.update.plan_complexity: Zero plan MU detected for uid %s' % study_instance_uid push_to_log(msg=msg)
def __init__(self, table_name, condition_str, unique=False, columns=None, group=1): """ :param table_name: 'Beams', 'DVHs', 'Plans', or 'Rxs' :type table_name: str :param condition_str: condition in SQL syntax :type condition_str: str :param unique: If set to True, only unique values stored :type unique: bool :param group: either 1 or 2 :type group: int """ table_name = table_name.lower() if table_name in {'beams', 'dvhs', 'plans', 'rxs'}: self.table_name = table_name self.condition_str = condition_str with DVH_SQL(group=group) as cnx: all_columns = cnx.get_column_names(table_name) if columns is not None: columns = set(all_columns).intersection(columns) # ensure provided columns exist in SQL table else: columns = all_columns for column in columns: if column not in {'roi_coord_string', 'distances_to_ptv'}: # ignored for memory since not used here self.cursor = cnx.query(self.table_name, column, self.condition_str) force_date = cnx.is_sqlite_column_datetime(self.table_name, column) # returns False for pgsql rtn_list = self.cursor_to_list(force_date=force_date) if unique: rtn_list = get_unique_list(rtn_list) setattr(self, column, rtn_list) # create property of QuerySQL based on SQL column name else: push_to_log(msg='QuerySQL: Table name in valid. Please select from Beams, DVHs, Plans, or Rxs.')
def calc_stats(data): """ Calculate a standard set of stats for DVHA :param data: a list or numpy 1D array of numbers :type data: list :return: max, 75%, median, mean, 25%, and min of data :rtype: list """ data = [x for x in data if x != 'None'] try: data_np = np.array(data) rtn_data = [ np.max(data_np), np.percentile(data_np, 75), np.median(data_np), np.mean(data_np), np.percentile(data_np, 25), np.min(data_np) ] except Exception as e: rtn_data = [0, 0, 0, 0, 0, 0] msg = "tools.utilities.calc_stats: received non-numerical data" push_to_log(e, msg=msg) return rtn_data
def __validate_input(rt_dose): """Ensure provided input is either an RT Dose pydicom.FileDataset or a file_path to one""" if type(rt_dose) is pydicom.FileDataset: if rt_dose.Modality.lower() == 'rtdose': return rt_dose msg = "DoseGrid.__validate_input: The provided pydicom.FileDataset is not RTDOSE" push_to_log(msg=msg) return elif isfile(rt_dose): try: rt_dose_ds = pydicom.read_file(rt_dose) if rt_dose_ds.Modality.lower() == 'rtdose': return rt_dose_ds msg = 'DoseGrid.__validate_input: ' \ 'The provided file_path points to a DICOM file, but it is not an RT Dose file.' push_to_log(msg=msg) except Exception as e: msg = 'DoseGrid.__validate_input: ' \ 'The provided input is neither a pydicom.FileDataset nor could it be read by pydicom.' push_to_log(e, msg=msg) return
def do_association(self): # associate appropriate rtdose files to plans for file_index, dose_file in enumerate(self.dicom_files['rtdose']): dose_tag_values = self.dicom_tag_values[dose_file] ref_plan_uid = dose_tag_values['ref_sop_instance']['uid'] study_uid = dose_tag_values['study_instance_uid'] mrn = dose_tag_values['mrn'] if mrn in self.plan_file_sets.keys(): if study_uid in self.plan_file_sets[mrn].keys(): for plan_file_set in self.plan_file_sets[mrn][ study_uid].values(): plan_uid = plan_file_set['rtplan']['sop_instance_uid'] if plan_uid == ref_plan_uid: self.dicom_tag_values[dose_file]['matched'] = True plan_file_set['rtdose'] = { 'file_path': dose_file, 'sop_instance_uid': dose_tag_values['sop_instance_uid'] } if 'rtdose' in self.dicom_file_paths[ plan_uid].keys(): self.dicom_file_paths[plan_uid][ 'rtdose'].append(dose_file) else: self.dicom_file_paths[plan_uid]['rtdose'] = [ dose_file ] else: msg = "%s: StudyInstanceUID from DICOM-RT Dose file " \ "could not be matched to any DICOM-RT Plan" % dose_file push_to_log(msg=msg) else: msg = "%s: PatientID from DICOM-RT Dose file could not be " \ "matched to any DICOM-RT Plan" % dose_file push_to_log(msg=msg) # associate appropriate rtstruct files to plans for mrn_index, mrn in enumerate(list(self.plan_file_sets)): for study_uid in list(self.plan_file_sets[mrn]): for plan_uid, plan_file_set in self.plan_file_sets[mrn][ study_uid].items(): plan_file = plan_file_set['rtplan']['file_path'] ref_struct_uid = self.dicom_tag_values[plan_file][ 'ref_sop_instance']['uid'] for struct_file in self.dicom_files['rtstruct']: struct_uid = self.dicom_tag_values[struct_file][ 'sop_instance_uid'] if struct_uid == ref_struct_uid: self.dicom_tag_values[plan_file]['matched'] = True plan_file_set['rtstruct'] = { 'file_path': struct_file, 'sop_instance_uid': struct_uid } if 'rtstruct' in self.dicom_file_paths[ plan_uid].keys(): self.dicom_file_paths[plan_uid][ 'rtstruct'].append(struct_file) else: self.dicom_file_paths[plan_uid]['rtstruct'] = [ struct_file ] # find unmatched structure and dose files, pair by StudyInstanceUID to plan if plan doesn't have struct/dose for dcm_file, tags in self.dicom_tag_values.items(): modality = tags['modality'] if not tags['matched'] and modality in {'rtstruct', 'rtdose'}: for mrn_index, mrn in enumerate(list(self.plan_file_sets)): for study_uid in list(self.plan_file_sets[mrn]): for plan_uid, plan_file_set in self.plan_file_sets[ mrn][study_uid].items(): if study_uid == tags['study_instance_uid']: if modality not in plan_file_set.keys(): self.dicom_file_paths[plan_uid][ modality] = [dcm_file] else: self.dicom_file_paths[plan_uid][ modality].append(dcm_file) # Check for multiple dose and structure files for plan_uid in list(self.dicom_file_paths): for file_type in ['rtstruct', 'rtdose']: files = self.dicom_file_paths[plan_uid][file_type] if len(files) > 1: timestamps = [ self.dicom_tag_values[f]['timestamp'] for f in files ] # os.path.getmtime() if file_type == 'rtdose': dose_sum_types = [ self.dicom_tag_values[f]['dose_sum_type'] for f in files ] non_ignored_types = [ x for x in dose_sum_types if x != 'IGNORED' ] dose_type_count = len(set(non_ignored_types)) if dose_type_count == 1: dose_sum_type = non_ignored_types[0] indices = [ i for i, sum_type in enumerate(dose_sum_types) if sum_type == dose_sum_type ] timestamps = [timestamps[i] for i in indices ] # timestamps of file_indices else: timestamps = [] # for dose files, if no dose_sum_type is found and there are multiple files, ignore all of them if len(timestamps): final_index = timestamps.index( max(timestamps)) # get the latest file index self.dicom_file_paths[plan_uid][file_type] = [ files[final_index] ] else: self.dicom_file_paths[plan_uid][file_type] = []
def parser(self, file_path, msg): wx.CallAfter(pub.sendMessage, "pre_import_progress_update", msg=msg) file_name = os.path.basename(file_path) file_ext = os.path.splitext(file_path)[1] if file_ext and file_ext.lower() != '.dcm' or file_name.lower( ) == 'dicomdir': ds = None else: try: ds = dicom.read_file(file_path, stop_before_pixels=True, force=True) except InvalidDicomError: ds = None if ds is not None: if not self.is_data_set_valid(ds): msg = 'Cannot parse %s\nOne of these tags is missing: %s' % ( file_path, ', '.join(self.req_tags)) push_to_log(msg=msg) else: modality = ds.Modality.lower() timestamp = os.path.getmtime(file_path) dose_sum_type = str(getattr(ds, 'DoseSummationType', None)).upper() dose_sum_type = dose_sum_type if dose_sum_type in [ 'PLAN', 'BRACHY' ] else 'IGNORED' self.dicom_tag_values[file_path] = { 'timestamp': timestamp, 'study_instance_uid': ds.StudyInstanceUID, 'sop_instance_uid': ds.SOPInstanceUID, 'patient_name': ds.PatientName, 'mrn': ds.PatientID, 'modality': modality, 'matched': False, 'dose_sum_type': dose_sum_type } if modality not in self.file_types: if ds.StudyInstanceUID not in self.other_dicom_files.keys( ): self.other_dicom_files[ds.StudyInstanceUID] = [] self.other_dicom_files[ds.StudyInstanceUID].append( file_path) # Store these to move after import else: self.dicom_files[modality].append(file_path) # All RT Plan files need to be found first if modality == 'rtplan' and hasattr( ds, 'ReferencedStructureSetSequence'): uid = ds.ReferencedStructureSetSequence[ 0].ReferencedSOPInstanceUID mrn = self.dicom_tag_values[file_path]['mrn'] self.uid_to_mrn[uid] = ds.PatientID self.dicom_tag_values[file_path][ 'ref_sop_instance'] = { 'type': 'struct', 'uid': uid } study_uid = ds.StudyInstanceUID plan_uid = ds.SOPInstanceUID if mrn not in list(self.plan_file_sets): self.plan_file_sets[mrn] = {} if study_uid not in list(self.plan_file_sets[mrn]): self.plan_file_sets[mrn][study_uid] = {} self.plan_file_sets[mrn][study_uid][plan_uid] = { 'rtplan': { 'file_path': file_path, 'sop_instance_uid': plan_uid }, 'rtstruct': { 'file_path': None, 'sop_instance_uid': None }, 'rtdose': { 'file_path': None, 'sop_instance_uid': None } } if plan_uid not in self.dicom_file_paths.keys(): self.dicom_file_paths[plan_uid] = { key: [] for key in self.file_types + ['other'] } self.dicom_file_paths[plan_uid]['rtplan'] = [file_path] elif modality == 'rtdose': uid = ds.ReferencedRTPlanSequence[ 0].ReferencedSOPInstanceUID self.dicom_tag_values[file_path][ 'ref_sop_instance'] = { 'type': 'plan', 'uid': uid } else: self.dicom_tag_values[file_path][ 'ref_sop_instance'] = { 'type': None, 'uid': None }
def min_distances(study_instance_uid, roi_name, pre_calc=None): """ Recalculate the min, mean, median, and max PTV distances an roi based on data in the SQL DB. Optionally provide coordinates of combined PTV, return from get_treatment_volume_coord """ oar_coordinates_string = query( 'dvhs', 'roi_coord_string', "study_instance_uid = '%s' and roi_name = '%s'" % (study_instance_uid, roi_name)) treatment_volume_coord = pre_calc if treatment_volume_coord is None: with DVH_SQL() as cnx: ptv_coordinates_strings = cnx.query( 'dvhs', 'roi_coord_string', "study_instance_uid = '%s' and roi_type like 'PTV%%'" % study_instance_uid) ptvs = [ roi_form.get_planes_from_string(ptv[0]) for ptv in ptv_coordinates_strings ] treatment_volume_coord = roi_form.get_roi_coordinates_from_planes( roi_geom.union(ptvs)) oar_coordinates = roi_form.get_roi_coordinates_from_string( oar_coordinates_string[0][0]) treatment_volume_coord = sample_roi(treatment_volume_coord) oar_coordinates = sample_roi(oar_coordinates) try: data = roi_geom.min_distances_to_target(oar_coordinates, treatment_volume_coord) except MemoryError: try: treatment_volume_coord = sample_roi(treatment_volume_coord, max_point_count=3000) oar_coordinates = sample_roi(oar_coordinates, max_point_count=3000) data = roi_geom.min_distances_to_target(oar_coordinates, treatment_volume_coord) except Exception as e: msg = 'db.update.min_distances: Error reported for %s with study_instance_uid %s\n' \ 'Skipping PTV distance and DTH calculations for this ROI.' % (roi_name, study_instance_uid) push_to_log(e, msg=msg) data = None if data is not None: try: dth = roi_geom.dth(data) dth_string = ','.join(['%.3f' % num for num in dth]) data_map = { 'dist_to_ptv_min': round(float(np.min(data)), 2), 'dist_to_ptv_mean': round(float(np.mean(data)), 2), 'dist_to_ptv_median': round(float(np.median(data)), 2), 'dist_to_ptv_max': round(float(np.max(data)), 2), 'dth_string': dth_string } except MemoryError as e: msg = 'Error reported for %s with study_instance_uid %s\n' \ 'Skipping PTV distance and DTH calculations for this ROI.' % (roi_name, study_instance_uid) push_to_log(e, msg=msg) data_map = None if data_map: for key, value in data_map.items(): update_dvhs_table(study_instance_uid, roi_name, key, value)
def write_test(config=None, db_type='pgsql', group=1, table=None, column=None, value=None): try: if config: if db_type == 'pgsql' and ('dbname' not in list(config) or 'port' not in list(config)): return None cnx = DVH_SQL(config, db_type=db_type, group=group) else: cnx = DVH_SQL(group=group) except Exception as e: push_to_log(e, msg="Write Test: Connection to SQL could not be established") return {'write': False, 'delete': False} try: cnx.initialize_database() except Exception as e: push_to_log(e, msg="Write Test: DVH_SQL.initialize_database failed") if table is None: table = cnx.tables[-1] if column is None: column = 'mrn' if value is None: # Find a test value that does not exist in the database value_init = 'SqlTest_' i = 0 value = value_init + str(i) current_values = cnx.get_unique_values(table, column) while value in current_values: value = value_init + str(i) i += 1 condition_str = "%s = '%s'" % (column, value) insert_cmd = "INSERT INTO %s (%s) VALUES ('%s');" % (table, column, value) delete_cmd = "DELETE FROM %s WHERE %s;" % (table, condition_str) try: cnx.execute_str(insert_cmd) except Exception as e: push_to_log(e, msg="Write Test: SQL test insert command failed") try: test_return = cnx.query(table, column, condition_str) write_test_success = len(test_return) > 0 except Exception as e: write_test_success = False push_to_log(e, msg="Write Test: SQL query of test insert failed") if not write_test_success: delete_test_success = None else: try: cnx.execute_str(delete_cmd) test_return = cnx.query(table, column, condition_str) delete_test_success = len(test_return) == 0 except Exception as e: delete_test_success = False push_to_log(e, msg="Write Test: SQL delete command of test insert failed") try: cnx.close() except Exception as e: push_to_log(e, msg='Write Test: Failed to close SQL connection') return {'write': write_test_success, 'delete': delete_test_success}