def all_dx_to_file():
orig_stdout = sys.stdout
newfile = open(f'R:/groups/seeley/Mack/NP report parser/NACC backlog dx sections_{date_today}.txt', 'w')
sys.stdout = newfile
for f in pNum_files.values():
pNum = filepath_dict[f]
dx_sxn = parser_functions.get_dx_sxn(f)
all_content = parser_functions.open_file(f)
print(f'\n --- {pNum} --- \n')
primDx_list = parser_functions.get_PrimDx(dx_sxn)
print(f'Number of primary diagnoses: {len(primDx_list)}')
for counter, dx in enumerate(primDx_list, 1):
print(f'{counter}: {dx}')
contributingDx_list = parser_functions.get_ContributingDx(dx_sxn)
print(f'\nNumber of contributing diagnoses: {len(contributingDx_list)}')
for counter, dx in enumerate(contributingDx_list, 1):
print(f'{counter}: {dx}')
incidentalDx_list = parser_functions.get_IncidentalDx(dx_sxn)
print(f'\nNumber of incidental diagnoses: {len(incidentalDx_list)}')
for counter, dx in enumerate(incidentalDx_list, 1):
print(f'{counter}: {dx}')
sys.stdout = orig_stdout
newfile.close()
def check_microinfarcts(pNum):
dict_entry = pf.get_files([pNum])
filename = dict_entry[pNum]
#print(f'Filename: {filename}')
lines = pf.open_file(filename)
microinfarct_sxn = case_finder.get_microinfarct_section(lines)
lines = []
for line in microinfarct_sxn:
if 'FOUND' in line:
lines.append(line)
distinct_sentences_list = []
for line in lines:
distinct_sentences = line.split('.')
for sentence in distinct_sentences:
sentence = sentence.replace('FINDINGS:', '')
sentence = sentence.replace(r'\t', '')
sentence = sentence.strip()
if sentence == '':
continue
distinct_sentences_list.append(sentence)
#print(f'{pNum} sentences: {distinct_sentences_list}')
values = add_microinfarct_score(distinct_sentences_list)
microinfarct_values = dict(zip(microinfarct_columns, values))
#for key, value in microinfarct_values.items():
# print(f'{key}: {value}')
return list(microinfarct_values.values())
def examine_CTE():
file_list = parser_functions.get_files()
for pNum, filename in file_list.items():
lines = parser_functions.open_file(filename)
for line in lines:
if any(x in line for x in ('hronic traumatic', 'CTE')):
print(f'{pNum}: {line}')
def examine_grossObs():
cases_without_grossObs = []
for pNum in pNum_files.keys():
filename = pNum_files[pNum]
all_content = parser_functions.open_file(filename)
i = 0
for line in all_content:
line = line.upper()
if 'GROSS OBSERVATIONS' in line:
i += 1
if i == 0:
cases_without_grossObs.append(pNum)
for pNum in cases_without_grossObs:
print(pNum)
def get_all_dx():
for f in pNum_files.values():
pNum = filepath_dict[f]
dx_sxn = parser_functions.get_dx_sxn(f)
all_content = parser_functions.open_file(f)
print(f'\n --- {pNum} --- \n')
primDx_list = parser_functions.get_PrimDx(dx_sxn)
print(f'Number of primary diagnoses: {len(primDx_list)}')
for counter, dx in enumerate(primDx_list, 1):
print(f'{counter}: {dx}')
contributingDx_list = parser_functions.get_ContributingDx(dx_sxn)
print(f'\nNumber of contributing diagnoses: {len(contributingDx_list)}')
for counter, dx in enumerate(contributingDx_list, 1):
print(f'{counter}: {dx}')
incidentalDx_list = parser_functions.get_IncidentalDx(dx_sxn)
print(f'\nNumber of incidental diagnoses: {len(incidentalDx_list)}')
for counter, dx in enumerate(incidentalDx_list, 1):
print(f'{counter}: {dx}')
def examine_vbi():
pNum_list = []
for f in pNum_files.values():
print(f'Working on {f}...')
pNum = filepath_dict[f]
dx_sxn = parser_functions.get_dx_sxn(f)
all_content = parser_functions.open_file(f)
for line in all_content:
line = line.upper()
## Note: needs to account for 'MICROINFARCTION(S), AGE, WAS/WERE FOUND'
if 'MICROINFARCTION WAS FOUND' in line:
pNum_list.append(pNum)
if 'MICROINFARCT WAS FOUND' in line:
pNum_list.append(pNum)
if 'MICROINFARCTS WERE FOUND' in line:
pNum_list.append(pNum)
if 'MICROINFARCTIONS WERE FOUND' in line:
pNum_list.append(pNum)
pNum_list = sorted(set(pNum_list))
return pNum_list
def parser_rows():
file_dict = pf.get_files(pNums_to_check)
# Set working fields to column names spanning entire DDS
# Create empty dataframe with parser fields as columns
data = []
for pNum in file_dict.keys():
print(f' --- {pNum} ---')
filename = file_dict[pNum]
all_contents = pf.open_file(filename)
dx_sxn = pf.get_dx_sxn(filename)
grossObs = pf.get_grossObs(filename)
site = 'UCSF NDBB'
author = pf.get_author(pNum)
ADNC_dict = pf.get_ADNC(dx_sxn)
Thal_phase = ADNC_dict['Thal Phase']
AD_Braak = ADNC_dict['Braak Stage']
AD_CERAD_NP = ADNC_dict['CERAD NP Score']
AD_CERAD_DP = ADNC_dict['CERAD DP Score']
NIAReag = ADNC_dict['NIA-Reagan']
CAA = pf.get_CAA(dx_sxn)
ADNC_level = ADNC_dict['ADNC level']
LBD = pf.get_lbd_stage(dx_sxn)
PD_Braak = pf.get_PDBraak(dx_sxn)
ATAC = pf.get_ATAC(dx_sxn)
#CTE = pf.get_CTE(dx_sxn)
#HS = 'NA'
#HS_laterality = 'NA'
Arterio = pf.get_arterio(dx_sxn)
Athero = pf.get_athero(grossObs)
#TDP_proteinopathy = 'NA'
AGD = pf.get_AGD(dx_sxn)
HD = pf.get_huntington(dx_sxn)
microinfarcts = check_microinfarcts(pNum)
parser_values = [
pNum, site, author, Thal_phase, AD_Braak, AD_CERAD_NP, AD_CERAD_DP,
NIAReag, CAA, ADNC_level, LBD, PD_Braak, ATAC, Arterio, Athero,
AGD, HD
] + microinfarcts
data.append(parser_values)
working_data = dict(zip(working_fields, parser_values))
print(working_data)
"""
primDx_list = pf.get_PrimDx(dx_sxn)
print(f'\nNumber of primary diagnoses: {len(primDx_list)}')
for counter, dx in enumerate(primDx_list, 1):
print(f'{counter}: {dx}')
contributingDx_list = pf.get_ContributingDx(dx_sxn)
print(f'\nNumber of contributing diagnoses: {len(contributingDx_list)}')
for counter, dx in enumerate(contributingDx_list, 1):
print(f'{counter}: {dx}')
incidentalDx_list = pf.get_IncidentalDx(dx_sxn)
print(f'\nNumber of incidental diagnoses: {len(incidentalDx_list)}')
for counter, dx in enumerate(incidentalDx_list, 1):
print(f'{counter}: {dx}')
"""
working_df = pd.DataFrame(data, columns=working_fields)
#print(working_df)
book = load_workbook(error_checking_sheet)
with pd.ExcelWriter(error_checking_sheet, engine='openpyxl') as writer:
writer.book = book
working_df.to_excel(writer, 'parser values')
return working_df
def examine_vbi_regions():
all_vbi_terms = []
distinct_sentences_list = []
for pNum in examine_vbi():
dict_entry = parser_functions.get_files([pNum])
filename = dict_entry[pNum]
print(f'Filename: {filename}')
lines = parser_functions.open_file(filename)
lines = get_microinfarct_section(lines)
print(lines)
for line in lines:
line = line.upper()
if any(x in line for x in microinfarct_phrases):
print(line)
distinct_sentences = line.split('.')
for sentence in distinct_sentences:
distinct_sentences_list.append(sentence)
print(f'Sentence: {sentence}')
distinct_clauses = sentence.split('AND')
for clause in distinct_clauses:
print(f'Clause: {clause}')
distinct_terms = clause.split(',')
for term in distinct_terms:
print(f'Term: {term}')
all_vbi_terms.append(term)
clean_vbi_terms = []
before_vbi_terms = []
gray_matter_terms = []
white_matter_terms = []
before_region_terms = []
after_region_terms = []
for term in all_vbi_terms:
term = term.strip()
if any(x in term for x in ['GRAY', 'CORTEX OF', '(CORTEX)', '(GRAY)', '(GRAY']):
gray_matter_terms.append(term)
if any(x in term for x in ['SUBCORTICAL', '(SUBCORTICAL', '(SUBCORTICAL)''WHITE MATTER', 'MATTER)']):
white_matter_terms.append(term)
if 'NO MICROINFARCTION' in term:
continue
elif r'FINDINGS:\t' in term:
terms = term.split(r'FINDINGS:\t')
if 'FOUND IN' in terms[1]:
terms = term[1].split('FOUND IN')
before_region_terms.append(terms[0])
for article in ['THE ', 'A ']:
if article in terms[1]:
terms = terms[1].split(article)
clean_vbi_terms.append(terms[1].strip())
else:
clean_vbi_terms.append(terms[1].strip())
elif 'FOUND IN' in term:
terms = term.split('FOUND IN')
before_vbi_terms.append(terms[0])
has_article = 0
for article in ['THE ', 'A ']:
if article in terms[1]:
has_article = 1
terms = terms[1].split(article)
clean_vbi_terms.append(term.strip())
if has_article == 0:
clean_vbi_terms.append(terms[1].strip())
elif term == '':
continue
else:
clean_vbi_terms.append(term.strip())
clean_vbi_terms = set(clean_vbi_terms)
before_vbi_terms = set(before_vbi_terms)
gray_matter_terms = sorted(set(gray_matter_terms))
white_matter_terms = sorted(set(white_matter_terms))
#for term in sorted(clean_vbi_terms):
# print(term)
#print(len(clean_vbi_terms))
#for term in sorted(before_vbi_terms):
# print(term)
#print(len(before_vbi_terms))
print(f'Gray matter terms:')
for term in gray_matter_terms:
print(term)
print(f'White matter terms:')
for term in white_matter_terms:
print(term)
for sentence in set(distinct_sentences_list):
print(sentence)
print(len(set(distinct_sentences_list)))
def weird_vbi_to_file():
current_time = time.strftime('%d-%m-%Y %H:%M:%S')
orig_stdout = sys.stdout
newfile = open(f'R:/groups/seeley/Mack/NP report parser/Abnormal VBI sxns_{date_today}.txt', 'w')
sys.stdout = newfile
print(f'| Cases with Abnormal VBI |\nRun from working_pNums file on {current_time}\n\n')
vbi_list = []
gross_infarct_list = []
check_grossObs_list = []
check_dx_list = []
for f in pNum_files.values():
pNum = filepath_dict[f]
try:
dx_sxn = parser_functions.get_dx_sxn(f)
all_content = parser_functions.open_file(f)
except UnboundLocalError:
print(f'{pNum}: Diagnostic sxn index referenced before assignment')
hx_sxn = {}
try:
gross_obs = parser_functions.get_grossObs(f)
gross_obs = [item.upper() for item in gross_obs]
except TypeError:
print(f'{pNum}: No gross obs')
for line in all_content:
if '(H & E)' in line:
if 'Other significant pathology' not in line:
hx_sxn[line] = all_content[all_content.index(line) + 2]
else:
hx_sxn[line] = all_content[all_content.index(line) + 1]
gross_infarcts = {}
if gross_obs:
for line in gross_obs:
if any(x in line for x in ['TERRITORIAL INFARCTS', 'LACUNAR INFARCTS']):
if any(x in line for x in ['ABSENT', 'NONE']):
continue
else:
line = line.split(':')
gross_infarcts[line[0]] = line[1]
if gross_infarcts:
vbi_list.append(pNum)
gross_infarct_list.append(pNum)
print(f'--- {pNum} ---\n')
for key, value in gross_infarcts.items():
print(f'{key}: {value}\n')
vbi_lineItem = ''
for line in dx_sxn:
if 'VASCULAR BRAIN INJURY' in line:
vbi_lineItem = line
if vbi_lineItem != '':
for key in hx_sxn:
if 'Microinfarcts' in key:
if all(x not in hx_sxn[key].upper() for x in ['SCARCE', 'MODERATE', 'SEVERE']):
#print(f'{key}:\n{hx_sxn[key]}\n')
vbi_list.append(pNum)
vbi_list = sorted(set(vbi_list))
gross_infarct_list = sorted(set(gross_infarct_list))
print('Weird VBI P# List:')
for pNum in vbi_list:
print(f'{pNum}')
print(len(vbi_list))
print('Gross Infarct List:')
for pNum in gross_infarct_list:
print(f'{pNum}')
print(len(gross_infarct_list))
sys.stdout = orig_stdout
newfile.close()
def vbi_dx_to_file():
current_time = time.strftime('%d-%m-%Y %H:%M:%S')
orig_stdout = sys.stdout
newfile = open(f'R:/groups/seeley/Mack/NP report parser/VBI sxns_{date_today}.txt', 'w')
sys.stdout = newfile
print(f'| Cases with VBI |\nRun from all files {current_time}\n\n')
vbi_list = []
for f in pNum_files.values():
print(f'Working on {f}...')
pNum = filepath_dict[f]
dx_sxn = parser_functions.get_dx_sxn(f)
all_content = parser_functions.open_file(f)
hx_sxn = {}
try:
gross_obs = parser_functions.get_grossObs(f)
gross_obs = [item.upper() for item in gross_obs]
except TypeError:
print(f'No gross obs found for {pNum}')
for line in all_content:
if '(H & E)' in line:
if 'Other significant pathology' not in line:
hx_sxn[line] = all_content[all_content.index(line) + 2]
else:
hx_sxn[line] = all_content[all_content.index(line) + 1]
gross_infarcts = {}
if gross_obs:
for line in gross_obs:
if any(x in line for x in ['TERRITORIAL INFARCTS', 'LACUNAR INFARCTS']):
if any(x in line for x in ['ABSENT', 'NONE']):
continue
else:
line = line.split(':')
gross_infarcts[line[0]] = line[1]
vbi_lineItem = ''
for line in dx_sxn:
if 'VASCULAR BRAIN INJURY' in line:
vbi_lineItem = line
vbi_list.append(pNum)
if vbi_lineItem != '':
print(f'--- {pNum} ---\n')
for key in hx_sxn:
if 'Microinfarcts' in key:
print(f'{key}:\n{hx_sxn[key]}\n')
if 'Other significant pathology' in key:
print(f'{key}:\n{hx_sxn[key]}\n')
if gross_infarcts:
for key, value in gross_infarcts.items():
print(f'{key}: {value}')
print('P# List:')
for pNum in vbi_list:
print(f'{pNum}')
print(len(vbi_list))
sys.stdout = orig_stdout
newfile.close()