def recalculate_plan_complexities_from_beams():
with DVH_SQL() as cnx:
uids = cnx.get_unique_values('Plans', 'study_instance_uid')
for uid in uids:
try:
condition = "study_instance_uid = '%s'" % uid
beam_complexities = cnx.query(
'Beams', 'fx_count, complexity, fx_grp_number', condition)
complexity = {}
fx_counts = {}
for row in beam_complexities:
fx_count, beam_complexity, fx_group_number = tuple(row)
if fx_group_number not in complexity:
complexity[fx_group_number] = 0.0
fx_counts[fx_group_number] = fx_count
complexity[fx_group_number] += beam_complexity
total_fx = float(sum([fx for fx in fx_counts.values()]))
plan_complexity = sum([
c * fx_counts[fx_grp] for fx_grp, c in complexity.items()
]) / total_fx
except Exception as e:
msg = "tools.utilities.recalculate_plan_complexities_from_beams: failed on uid = %s" % uid
push_to_log(e, msg=msg)
plan_complexity = None
if plan_complexity is not None:
cnx.update('Plans', 'complexity', plan_complexity, condition)
def echo_sql_db(config=None, db_type='pgsql', group=1):
"""
Echo the database using stored or provided credentials
:param config: database login credentials
:type config: dict
:param db_type: either 'pgsql' or 'sqlite'
:type db_type: str
:param group: either group 1 or 2
:type group: int
:return: True if connection could be established
:rtype: bool
"""
try:
if config:
if db_type == 'pgsql' and ('dbname' not in list(config) or 'port' not in list(config)):
return False
cnx = DVH_SQL(config, db_type=db_type, group=group)
else:
cnx = DVH_SQL(group=group)
cnx.close()
return True
except Exception as e:
if type(e) not in [psycopg2.OperationalError, sqlite3.OperationalError]:
push_to_log(e, msg='Unknown Error during SQL Echo')
return False
def delete_file(file_path):
try:
if isfile(file_path):
unlink(file_path)
elif isdir(file_path):
shutil.rmtree(file_path)
except Exception as e:
push_to_log(e, msg='tools.utilities.delete_file: %s' % file_path)
def get_physician_from_uid(uid):
with DVH_SQL() as cnx:
results = cnx.query('Plans', 'physician', "study_instance_uid = '" + uid + "'")
if len(results) > 1:
msg = 'roi_name_manager.get_physician_from_uid: multiple plans with this study_instance_uid exist: %s' % uid
push_to_log(msg=msg)
return str(results[0][0])
def is_options_file_valid(self):
try:
current_checksum = self.calculate_checksum()
stored_checksum = self.load_stored_checksum()
if current_checksum == stored_checksum:
return True
except Exception as e:
msg = 'Options.is_options_file_valid: Corrupted options file detected. Loading default options.'
push_to_log(e, msg=msg)
return False
def sync_spin_buttons(self):
if self.x_axis:
index = self.choices.index(self.x_axis)
self.spin_button_x_axis.SetValue(len(self.choices) - 1 - index)
index = self.choices.index(self.y_axis)
self.spin_button_y_axis.SetValue(len(self.choices) - 1 - index)
else:
msg = 'RegressionFrame.sync_spin_buttons: x-axis choice is empty.'
push_to_log(msg=msg)
def edit_study_uid(abs_file_path, study_uid):
"""
Change the StudyInstanceUID of a DICOM file
:param abs_file_path: absolute file path of the DICOM file
:param study_uid: new StudyInstanceUID
"""
try:
ds = pydicom.read_file(abs_file_path, force=True)
ds.StudyInstanceUID = study_uid
ds.save_as(abs_file_path)
except Exception as e:
push_to_log(e, abs_file_path)
def set_option(self, attr, value):
"""
Change or create an option value
:param attr: name of option
:type attr: str
:param value: value of option
"""
if not hasattr(self, attr):
msg = 'Options.set_option: %s did not previously exist' % attr
push_to_log(msg=msg)
setattr(self, attr, value)
self.is_edited = True
def load(self):
self.is_edited = False
if isfile(OPTIONS_PATH) and self.is_options_file_valid:
try:
with open(OPTIONS_PATH, 'rb') as infile:
loaded_options = pickle.load(infile)
self.upgrade_options(loaded_options)
except Exception as e:
msg = 'Options.load: Options file corrupted. Loading default options.'
push_to_log(e, msg=msg)
loaded_options = {}
for key, value in loaded_options.items():
if hasattr(self, key):
setattr(self, key, value)
def load_roi_types_from_file(self, physician):
if self.is_physician(physician):
file_path = os.path.join(PREF_DIR, 'physician_%s.rtype' % physician)
if os.path.isfile(file_path):
with open(file_path, 'r') as doc:
for line in doc:
if not line:
continue
if line.count(':') == 1:
try:
physician_roi, roi_type = tuple(line.split(':'))
self.physicians[physician].rois[physician_roi.strip()].roi_type = roi_type.strip()
except Exception as e:
msg = 'DatabaseROIs.load_roi_types_from_file: ' \
'Could not import %s roi_type for physician %s' % (physician_roi, physician)
push_to_log(e, msg=msg)
def delete_column(self, column):
"""
Delete the specified column data and the layout
:param column: column to be deleted
:type column: str
"""
if column in self.keys:
index = self.columns.index(column)
if self.layout:
try:
self.layout.DeleteColumn(index)
except Exception as e:
msg = 'DataTable.delete_column: Could not delete column in layout'
push_to_log(e, msg=msg)
self.data.pop(column)
self.columns.pop(index)
def beam_complexity(cnx, study_instance_uid):
"""
:param cnx: connection to DVHA SQL database
:type cnx: DVH_SQL
:param study_instance_uid: study_instance_uid in SQL database
:type study_instance_uid: str
"""
rt_plan_query = cnx.query('DICOM_Files', 'folder_path, plan_file',
"study_instance_uid = '%s'" %
study_instance_uid)[0]
rt_plan_file_path = join_path(rt_plan_query[0], rt_plan_query[1])
rt_plan = dicom.read_file(rt_plan_file_path)
for beam_num, beam in enumerate(rt_plan.BeamSequence):
try:
condition = "study_instance_uid = '%s' and beam_number = '%s'" % (
study_instance_uid, (beam_num + 1))
meterset = float(cnx.query('Beams', 'beam_mu', condition)[0][0])
mlca_data = BeamAnalyzer(beam, meterset, ignore_zero_mu_cp=True)
mlc_keys = ['area', 'x_perim', 'y_perim', 'cmp_score', 'cp_mu']
summary_stats = {
key: calc_stats(mlca_data.summary[key])
for key in mlc_keys
}
column_vars = {
'area': 'area',
'x_perim': 'x_perim',
'y_perim': 'y_perim',
'complexity': 'cmp_score',
'cp_mu': 'cp_mu'
}
stat_map = {'min': 5, 'mean': 3, 'median': 2, 'max': 0}
for c in list(column_vars):
for s in list(stat_map):
value = summary_stats[column_vars[c]][stat_map[s]]
column = "%s_%s" % (c, s)
cnx.update('Beams', column, value, condition)
cnx.update('Beams', 'complexity',
np.sum(mlca_data.summary['cmp_score']), condition)
except Exception as e:
msg = 'db.update.beam_complexity: MLC Analyzer fail for beam number %s and uid %s' % \
((beam_num+1), study_instance_uid)
push_to_log(e, msg=msg)
def update(self, table_name, column, value, condition_str):
"""
Change the data in the database.
:param table_name: 'DVHs', 'Plans', 'Rxs', 'Beams', or 'DICOM_Files'
:type table_name: str
:param column: SQL column to be updated
:type column: str
:param value: value to be set
:type value: str or float or int
:param condition_str: a condition in SQL syntax
:type condition_str: str
"""
try:
float(value)
value_is_numeric = True
except ValueError:
value_is_numeric = False
if '::date' in str(value):
amend_type = [
'', '::date'
][self.db_type == 'pgsql'] # sqlite3 does not support ::date
value = "'%s'%s" % (
value.strip('::date'), amend_type
) # augment value for postgresql date formatting
elif value_is_numeric:
value = str(value)
elif 'null' == str(value.lower()):
value = "NULL"
else:
value = "'%s'" % str(value) # need quotes to input a string
update = "Update %s SET %s = %s WHERE %s" % (table_name, column, value,
condition_str)
try:
self.cursor.execute(update)
self.cnx.commit()
except Exception as e:
push_to_log(e, msg="Database update failure!")
def plan_complexity(cnx, study_instance_uid):
"""
:param cnx: connection to DVHA SQL database
:type cnx: DVH_SQL
:param study_instance_uid: study_instance_uid in SQL database
:type study_instance_uid: str
"""
condition = "study_instance_uid = '%s'" % study_instance_uid
beam_data = query('Beams', 'complexity, beam_mu', condition)
scores = [row[0] for row in beam_data]
include = [i for i, score in enumerate(scores) if score]
scores = [score for i, score in enumerate(scores) if i in include]
beam_mu = [row[1] for i, row in enumerate(beam_data) if i in include]
plan_mu = np.sum(beam_mu)
if plan_mu:
complexity = np.sum(np.multiply(scores, beam_mu)) / plan_mu
cnx.update('Plans', 'complexity', complexity,
"study_instance_uid = '%s'" % study_instance_uid)
else:
msg = 'db.update.plan_complexity: Zero plan MU detected for uid %s' % study_instance_uid
push_to_log(msg=msg)
def __init__(self, table_name, condition_str, unique=False, columns=None, group=1):
"""
:param table_name: 'Beams', 'DVHs', 'Plans', or 'Rxs'
:type table_name: str
:param condition_str: condition in SQL syntax
:type condition_str: str
:param unique: If set to True, only unique values stored
:type unique: bool
:param group: either 1 or 2
:type group: int
"""
table_name = table_name.lower()
if table_name in {'beams', 'dvhs', 'plans', 'rxs'}:
self.table_name = table_name
self.condition_str = condition_str
with DVH_SQL(group=group) as cnx:
all_columns = cnx.get_column_names(table_name)
if columns is not None:
columns = set(all_columns).intersection(columns) # ensure provided columns exist in SQL table
else:
columns = all_columns
for column in columns:
if column not in {'roi_coord_string', 'distances_to_ptv'}: # ignored for memory since not used here
self.cursor = cnx.query(self.table_name,
column,
self.condition_str)
force_date = cnx.is_sqlite_column_datetime(self.table_name, column) # returns False for pgsql
rtn_list = self.cursor_to_list(force_date=force_date)
if unique:
rtn_list = get_unique_list(rtn_list)
setattr(self, column, rtn_list) # create property of QuerySQL based on SQL column name
else:
push_to_log(msg='QuerySQL: Table name in valid. Please select from Beams, DVHs, Plans, or Rxs.')
def calc_stats(data):
"""
Calculate a standard set of stats for DVHA
:param data: a list or numpy 1D array of numbers
:type data: list
:return: max, 75%, median, mean, 25%, and min of data
:rtype: list
"""
data = [x for x in data if x != 'None']
try:
data_np = np.array(data)
rtn_data = [
np.max(data_np),
np.percentile(data_np, 75),
np.median(data_np),
np.mean(data_np),
np.percentile(data_np, 25),
np.min(data_np)
]
except Exception as e:
rtn_data = [0, 0, 0, 0, 0, 0]
msg = "tools.utilities.calc_stats: received non-numerical data"
push_to_log(e, msg=msg)
return rtn_data
def __validate_input(rt_dose):
"""Ensure provided input is either an RT Dose pydicom.FileDataset or a file_path to one"""
if type(rt_dose) is pydicom.FileDataset:
if rt_dose.Modality.lower() == 'rtdose':
return rt_dose
msg = "DoseGrid.__validate_input: The provided pydicom.FileDataset is not RTDOSE"
push_to_log(msg=msg)
return
elif isfile(rt_dose):
try:
rt_dose_ds = pydicom.read_file(rt_dose)
if rt_dose_ds.Modality.lower() == 'rtdose':
return rt_dose_ds
msg = 'DoseGrid.__validate_input: ' \
'The provided file_path points to a DICOM file, but it is not an RT Dose file.'
push_to_log(msg=msg)
except Exception as e:
msg = 'DoseGrid.__validate_input: ' \
'The provided input is neither a pydicom.FileDataset nor could it be read by pydicom.'
push_to_log(e, msg=msg)
return
def do_association(self):
# associate appropriate rtdose files to plans
for file_index, dose_file in enumerate(self.dicom_files['rtdose']):
dose_tag_values = self.dicom_tag_values[dose_file]
ref_plan_uid = dose_tag_values['ref_sop_instance']['uid']
study_uid = dose_tag_values['study_instance_uid']
mrn = dose_tag_values['mrn']
if mrn in self.plan_file_sets.keys():
if study_uid in self.plan_file_sets[mrn].keys():
for plan_file_set in self.plan_file_sets[mrn][
study_uid].values():
plan_uid = plan_file_set['rtplan']['sop_instance_uid']
if plan_uid == ref_plan_uid:
self.dicom_tag_values[dose_file]['matched'] = True
plan_file_set['rtdose'] = {
'file_path':
dose_file,
'sop_instance_uid':
dose_tag_values['sop_instance_uid']
}
if 'rtdose' in self.dicom_file_paths[
plan_uid].keys():
self.dicom_file_paths[plan_uid][
'rtdose'].append(dose_file)
else:
self.dicom_file_paths[plan_uid]['rtdose'] = [
dose_file
]
else:
msg = "%s: StudyInstanceUID from DICOM-RT Dose file " \
"could not be matched to any DICOM-RT Plan" % dose_file
push_to_log(msg=msg)
else:
msg = "%s: PatientID from DICOM-RT Dose file could not be " \
"matched to any DICOM-RT Plan" % dose_file
push_to_log(msg=msg)
# associate appropriate rtstruct files to plans
for mrn_index, mrn in enumerate(list(self.plan_file_sets)):
for study_uid in list(self.plan_file_sets[mrn]):
for plan_uid, plan_file_set in self.plan_file_sets[mrn][
study_uid].items():
plan_file = plan_file_set['rtplan']['file_path']
ref_struct_uid = self.dicom_tag_values[plan_file][
'ref_sop_instance']['uid']
for struct_file in self.dicom_files['rtstruct']:
struct_uid = self.dicom_tag_values[struct_file][
'sop_instance_uid']
if struct_uid == ref_struct_uid:
self.dicom_tag_values[plan_file]['matched'] = True
plan_file_set['rtstruct'] = {
'file_path': struct_file,
'sop_instance_uid': struct_uid
}
if 'rtstruct' in self.dicom_file_paths[
plan_uid].keys():
self.dicom_file_paths[plan_uid][
'rtstruct'].append(struct_file)
else:
self.dicom_file_paths[plan_uid]['rtstruct'] = [
struct_file
]
# find unmatched structure and dose files, pair by StudyInstanceUID to plan if plan doesn't have struct/dose
for dcm_file, tags in self.dicom_tag_values.items():
modality = tags['modality']
if not tags['matched'] and modality in {'rtstruct', 'rtdose'}:
for mrn_index, mrn in enumerate(list(self.plan_file_sets)):
for study_uid in list(self.plan_file_sets[mrn]):
for plan_uid, plan_file_set in self.plan_file_sets[
mrn][study_uid].items():
if study_uid == tags['study_instance_uid']:
if modality not in plan_file_set.keys():
self.dicom_file_paths[plan_uid][
modality] = [dcm_file]
else:
self.dicom_file_paths[plan_uid][
modality].append(dcm_file)
# Check for multiple dose and structure files
for plan_uid in list(self.dicom_file_paths):
for file_type in ['rtstruct', 'rtdose']:
files = self.dicom_file_paths[plan_uid][file_type]
if len(files) > 1:
timestamps = [
self.dicom_tag_values[f]['timestamp'] for f in files
] # os.path.getmtime()
if file_type == 'rtdose':
dose_sum_types = [
self.dicom_tag_values[f]['dose_sum_type']
for f in files
]
non_ignored_types = [
x for x in dose_sum_types if x != 'IGNORED'
]
dose_type_count = len(set(non_ignored_types))
if dose_type_count == 1:
dose_sum_type = non_ignored_types[0]
indices = [
i for i, sum_type in enumerate(dose_sum_types)
if sum_type == dose_sum_type
]
timestamps = [timestamps[i] for i in indices
] # timestamps of file_indices
else:
timestamps = []
# for dose files, if no dose_sum_type is found and there are multiple files, ignore all of them
if len(timestamps):
final_index = timestamps.index(
max(timestamps)) # get the latest file index
self.dicom_file_paths[plan_uid][file_type] = [
files[final_index]
]
else:
self.dicom_file_paths[plan_uid][file_type] = []
def parser(self, file_path, msg):
wx.CallAfter(pub.sendMessage, "pre_import_progress_update", msg=msg)
file_name = os.path.basename(file_path)
file_ext = os.path.splitext(file_path)[1]
if file_ext and file_ext.lower() != '.dcm' or file_name.lower(
) == 'dicomdir':
ds = None
else:
try:
ds = dicom.read_file(file_path,
stop_before_pixels=True,
force=True)
except InvalidDicomError:
ds = None
if ds is not None:
if not self.is_data_set_valid(ds):
msg = 'Cannot parse %s\nOne of these tags is missing: %s' % (
file_path, ', '.join(self.req_tags))
push_to_log(msg=msg)
else:
modality = ds.Modality.lower()
timestamp = os.path.getmtime(file_path)
dose_sum_type = str(getattr(ds, 'DoseSummationType',
None)).upper()
dose_sum_type = dose_sum_type if dose_sum_type in [
'PLAN', 'BRACHY'
] else 'IGNORED'
self.dicom_tag_values[file_path] = {
'timestamp': timestamp,
'study_instance_uid': ds.StudyInstanceUID,
'sop_instance_uid': ds.SOPInstanceUID,
'patient_name': ds.PatientName,
'mrn': ds.PatientID,
'modality': modality,
'matched': False,
'dose_sum_type': dose_sum_type
}
if modality not in self.file_types:
if ds.StudyInstanceUID not in self.other_dicom_files.keys(
):
self.other_dicom_files[ds.StudyInstanceUID] = []
self.other_dicom_files[ds.StudyInstanceUID].append(
file_path) # Store these to move after import
else:
self.dicom_files[modality].append(file_path)
# All RT Plan files need to be found first
if modality == 'rtplan' and hasattr(
ds, 'ReferencedStructureSetSequence'):
uid = ds.ReferencedStructureSetSequence[
0].ReferencedSOPInstanceUID
mrn = self.dicom_tag_values[file_path]['mrn']
self.uid_to_mrn[uid] = ds.PatientID
self.dicom_tag_values[file_path][
'ref_sop_instance'] = {
'type': 'struct',
'uid': uid
}
study_uid = ds.StudyInstanceUID
plan_uid = ds.SOPInstanceUID
if mrn not in list(self.plan_file_sets):
self.plan_file_sets[mrn] = {}
if study_uid not in list(self.plan_file_sets[mrn]):
self.plan_file_sets[mrn][study_uid] = {}
self.plan_file_sets[mrn][study_uid][plan_uid] = {
'rtplan': {
'file_path': file_path,
'sop_instance_uid': plan_uid
},
'rtstruct': {
'file_path': None,
'sop_instance_uid': None
},
'rtdose': {
'file_path': None,
'sop_instance_uid': None
}
}
if plan_uid not in self.dicom_file_paths.keys():
self.dicom_file_paths[plan_uid] = {
key: []
for key in self.file_types + ['other']
}
self.dicom_file_paths[plan_uid]['rtplan'] = [file_path]
elif modality == 'rtdose':
uid = ds.ReferencedRTPlanSequence[
0].ReferencedSOPInstanceUID
self.dicom_tag_values[file_path][
'ref_sop_instance'] = {
'type': 'plan',
'uid': uid
}
else:
self.dicom_tag_values[file_path][
'ref_sop_instance'] = {
'type': None,
'uid': None
}
def min_distances(study_instance_uid, roi_name, pre_calc=None):
"""
Recalculate the min, mean, median, and max PTV distances an roi based on data in the SQL DB.
Optionally provide coordinates of combined PTV, return from get_treatment_volume_coord
"""
oar_coordinates_string = query(
'dvhs', 'roi_coord_string',
"study_instance_uid = '%s' and roi_name = '%s'" %
(study_instance_uid, roi_name))
treatment_volume_coord = pre_calc
if treatment_volume_coord is None:
with DVH_SQL() as cnx:
ptv_coordinates_strings = cnx.query(
'dvhs', 'roi_coord_string',
"study_instance_uid = '%s' and roi_type like 'PTV%%'" %
study_instance_uid)
ptvs = [
roi_form.get_planes_from_string(ptv[0])
for ptv in ptv_coordinates_strings
]
treatment_volume_coord = roi_form.get_roi_coordinates_from_planes(
roi_geom.union(ptvs))
oar_coordinates = roi_form.get_roi_coordinates_from_string(
oar_coordinates_string[0][0])
treatment_volume_coord = sample_roi(treatment_volume_coord)
oar_coordinates = sample_roi(oar_coordinates)
try:
data = roi_geom.min_distances_to_target(oar_coordinates,
treatment_volume_coord)
except MemoryError:
try:
treatment_volume_coord = sample_roi(treatment_volume_coord,
max_point_count=3000)
oar_coordinates = sample_roi(oar_coordinates, max_point_count=3000)
data = roi_geom.min_distances_to_target(oar_coordinates,
treatment_volume_coord)
except Exception as e:
msg = 'db.update.min_distances: Error reported for %s with study_instance_uid %s\n' \
'Skipping PTV distance and DTH calculations for this ROI.' % (roi_name, study_instance_uid)
push_to_log(e, msg=msg)
data = None
if data is not None:
try:
dth = roi_geom.dth(data)
dth_string = ','.join(['%.3f' % num for num in dth])
data_map = {
'dist_to_ptv_min': round(float(np.min(data)), 2),
'dist_to_ptv_mean': round(float(np.mean(data)), 2),
'dist_to_ptv_median': round(float(np.median(data)), 2),
'dist_to_ptv_max': round(float(np.max(data)), 2),
'dth_string': dth_string
}
except MemoryError as e:
msg = 'Error reported for %s with study_instance_uid %s\n' \
'Skipping PTV distance and DTH calculations for this ROI.' % (roi_name, study_instance_uid)
push_to_log(e, msg=msg)
data_map = None
if data_map:
for key, value in data_map.items():
update_dvhs_table(study_instance_uid, roi_name, key, value)
def write_test(config=None, db_type='pgsql', group=1, table=None, column=None, value=None):
try:
if config:
if db_type == 'pgsql' and ('dbname' not in list(config) or 'port' not in list(config)):
return None
cnx = DVH_SQL(config, db_type=db_type, group=group)
else:
cnx = DVH_SQL(group=group)
except Exception as e:
push_to_log(e, msg="Write Test: Connection to SQL could not be established")
return {'write': False, 'delete': False}
try:
cnx.initialize_database()
except Exception as e:
push_to_log(e, msg="Write Test: DVH_SQL.initialize_database failed")
if table is None:
table = cnx.tables[-1]
if column is None:
column = 'mrn'
if value is None:
# Find a test value that does not exist in the database
value_init = 'SqlTest_'
i = 0
value = value_init + str(i)
current_values = cnx.get_unique_values(table, column)
while value in current_values:
value = value_init + str(i)
i += 1
condition_str = "%s = '%s'" % (column, value)
insert_cmd = "INSERT INTO %s (%s) VALUES ('%s');" % (table, column, value)
delete_cmd = "DELETE FROM %s WHERE %s;" % (table, condition_str)
try:
cnx.execute_str(insert_cmd)
except Exception as e:
push_to_log(e, msg="Write Test: SQL test insert command failed")
try:
test_return = cnx.query(table, column, condition_str)
write_test_success = len(test_return) > 0
except Exception as e:
write_test_success = False
push_to_log(e, msg="Write Test: SQL query of test insert failed")
if not write_test_success:
delete_test_success = None
else:
try:
cnx.execute_str(delete_cmd)
test_return = cnx.query(table, column, condition_str)
delete_test_success = len(test_return) == 0
except Exception as e:
delete_test_success = False
push_to_log(e, msg="Write Test: SQL delete command of test insert failed")
try:
cnx.close()
except Exception as e:
push_to_log(e, msg='Write Test: Failed to close SQL connection')
return {'write': write_test_success, 'delete': delete_test_success}
