def __init__(self):
super().__init__()
self._validation_set = config.get_pipeline_config_item(
self.module_name(), 'validation_set_file', None)
self._df_notes_labeled_path = config.get_pipeline_config_item(
self.module_name(), 'input_note_file', None)
self._loaded_df = None
self._compare_df = None
self._orig_df = None
self._loaded_validation = None
self._loaded_validation_labels = None
self._loaded_validation_label_map = None
logger.log_info('Loading validation note labeling file')
self._loading_validation_labeling_file()
logger.log_info('DONE: Loading validation note labeling file')
logger.log_info('Loading NLP pipeline processed note files')
self._loading_note_files()
logger.log_info('DONE: NLP pipeline processed note files')
logger.log_info('Computing and outputting statistics')
self._do_statistics()
def __init__(self):
super().__init__()
if not google_tools.check_google_authenticated():
logger.log_error('User not authenticated for Google cloud access.')
self._output_note_file = config.get_pipeline_config_item(
self.module_name(), 'output_note_file',
'_labeled_lexicons.parquet')
self._cohort_file = config.get_pipeline_config_item(
self.module_name(), 'cohort_file', '')
self._lexicon_dir = config.get_pipeline_config_item(
self.module_name(), 'lexicon_dir', './used_lexicons')
self._lexicon_map = {'positive': {}, 'negative': {}}
self._lexicon_weights = {'positive': {}, 'negative': {}}
self._labeled_df = None
self._ignore_cat_list = config.get_pipeline_config_item(
self.module_name(), 'ignore_categories', [])
self._dump_all = config.get_pipeline_config_item(
self.module_name(), 'dump_all_notes', True)
self._ignore_cat_list = [
self._ignore_cat_list
] if not isinstance(self._ignore_cat_list,
collections.Sequence) else self._ignore_cat_list
self._parse_lexicons()
self._query_bigquery()
self._dump_df()
def __init__(self):
super().__init__()
self._output_note_file = config.get_pipeline_config_item(
self.module_name(), 'output_note_file',
'_negex_filtered_notes.parquet')
self._df_notes_labeled_path = config.get_pipeline_config_item(
self.module_name(), 'input_note_file', None)
self._catchall = config.get_pipeline_config_item(
self.module_name(), 'catchall', False)
self._loaded_df = None
self._filtered_cohort_df = None
self._labeled_df = None
logger.log_info('Loading note file')
self._load_note_input_file()
logger.log_info('DONE: Loading note file.')
logger.log_info('Replacing catchall labels.')
self._replace_catchall()
logger.log_info('DONE: Replacing catchall labels.')
logger.log_info('Dumping processed notes.')
self._dump_processed_df()
logger.log_info('DONE: Dumping processed notes.')
def __init__(self):
super().__init__()
self._output_note_file = config.get_pipeline_config_item(self.module_name(), 'output_note_file', '_negex_filtered_notes.parquet')
self._df_notes_labeled_path = config.get_pipeline_config_item(self.module_name(), 'input_note_file', None)
self._word_distance = config.get_pipeline_config_item(self.module_name(), 'word_distance', 10)
self._debug_check = config.get_pipeline_config_item(self.module_name(), 'debug_check', True)
#self._has_negation_flag = False
self._loaded_df = None
self._lexicon_dir = config.get_pipeline_config_item(self.module_name(), 'word_filter_dir', './word_cohort_filters')
self._lexicon_map = {}
self._njobs = config.get('njobs', multiprocessing.cpu_count())
logger.log_info('Loading note file')
self._load_note_input_file()
logger.log_info('DONE: Loading note file.')
logger.log_info('Parsing word filter lexicons.')
self._load_word_matching_lexicons()
logger.log_info('DONE: Parsing word filter lexicons.')
logger.log_info('Word filtering and cohort adaptions.')
self._change_cohort_mappings()
logger.log_info('DONE: Word filtering and cohort adaptions.')
logger.log_info('Dumping changed notes.')
self._dump_filtered_df()
logger.log_info('DONE: Dumping changed notes.')
def __init__(self):
super().__init__()
self._validation_set = config.get_pipeline_config_item(
self.module_name(), 'validation_set_file', None)
self._df_path_a = config.get_pipeline_config_item(
self.module_name(), 'file_prediction_a', None)
self._df_path_b = config.get_pipeline_config_item(
self.module_name(), 'file_prediction_b', None)
self._ignore_b = config.get_pipeline_config_item(
self.module_name(), 'ignore_b', False)
self._section_info__file = config.get_pipeline_config_item(
self.module_name(), 'section_info__file', None)
self._df_path_a_negated = config.get_pipeline_config_item(
self.module_name(), 'file_sentence_info_a', None)
self._df_path_b_negated = config.get_pipeline_config_item(
self.module_name(), 'file_sentence_info_b', None)
self._get_examples_for_categories = config.get_pipeline_config_item(
self.module_name(), 'get_examples_for_categories', None)
self._compare_dumping_dir_name = 'dumped_validaton_sentences_between__{}_and_{}'
self._required_tag_list = [
'POSITIVE_LEXICON_SENTENCES', 'POSITIVE_LEXICON_AFFIRMED_PHRASES',
'POSITIVE_LEXICON_NEGATED_PHRASES', 'NEGATIVE_LEXICON_SENTENCES',
'NEGATIVE_LEXICON_AFFIRMED_PHRASES',
'NEGATIVE_LEXICON_NEGATED_PHRASES'
]
self._df_a = None
self._df_b = None
self._df_sents_a = None
self._df_sents_b = None
self._loaded_validation = None
self._loaded_validation_labels = None
self._loaded_validation_label_map = None
logger.log_info('Processing validation file')
self._loading_validation_labeling_file()
logger.log_info('DONE: Processing validation file')
if self._section_info__file:
logger.log_info(
'Loading section file to include section information')
self._loading_section_info_file()
logger.log_info(
'DONE: Loading section file to include section information')
logger.log_info(
'Loading prediction and sentence info file for dataframes A and B to be compared.'
)
self._loading_note_files()
logger.log_info(
'DONE: Loading prediction and sentence info file for dataframes A and B to be compared.'
)
logger.log_info('Dumping sentences into folder')
self._dump_examples_for_comparison()
logger.log_info('DONE: Dumping sentences into folder')
def __init__(self):
super().__init__()
nltk.download('punkt')
self._output_note_file = config.get_pipeline_config_item(
self.module_name(), 'output_note_file',
'_negex_filtered_notes.parquet')
self._df_notes_labeled_path = config.get_pipeline_config_item(
self.module_name(), 'input_note_file', None)
self._negex_triggers = config.get_pipeline_config_item(
self.module_name(), 'negex_triggers', 'negex_trigger.txt')
self._add_fullstop_after_newline_uppercase = config.get_pipeline_config_item(
self.module_name(), 'add_fullstop_after_newline_uppercase', False)
self._keep_categories = config.get_pipeline_config_item(
self.module_name(), 'keep_categories', [])
self._dont_include_predicted_categories = config.get_pipeline_config_item(
self.module_name(), 'dont_include_predicted_categories', False)
self._debug_row_id = config.get_pipeline_config_item(
self.module_name(), 'debug_row_id', None)
self._loaded_df = None
self._pre_filtered_df = None
self._filtered_cohort_df = None
self._labeled_df = None
self._lexicon_dir = config.get_pipeline_config_item(
self.module_name(), 'lexicon_dir', './used_lexicons')
self._lexicon_map = {'positive': {}}
self._debug_check = config.get_pipeline_config_item(
self.module_name(), 'debug_check', False)
self._njobs = config.get('njobs', multiprocessing.cpu_count())
logger.log_info('Loading note file')
self._load_note_input_file()
logger.log_info('DONE: Loading note file.')
logger.log_info('Parsing Negex triggers.')
self._load_negex_triggers()
logger.log_info('DONE: Parsing Negex triggers.')
logger.log_info('Negex note filtering.')
self._parse_lexicons()
self._check_note_negations()
logger.log_info('DONE: Negex note filtering.')
logger.log_info('New cohort labeling.')
self._label_improve_cohort()
logger.log_info('DONE: New cohort labeling.')
logger.log_info('Dumping filtered notes.')
self._dump_filtered_df()
logger.log_info('DONE: Dumping filtered notes.')
def __init__(self):
super().__init__()
self._validation_set = config.get_pipeline_config_item(
self.module_name(), 'validation_set_file', None)
self._df_notes_labeled_paths = config.get_pipeline_config_item(
self.module_name(), 'input_note_files', [])
self._loaded_df = []
self._compare_df = None
self._orig_df = None
self._loaded_validation = None
self._loaded_validation_labels = None
self._loaded_validation_label_map = None
logger.log_info('Loading validation note labeling file')
self._loading_validation_labeling_file()
logger.log_info('DONE: Loading validation note labeling file')
logger.log_info('Loading NLP pipeline processed note files')
self._loading_note_files()
logger.log_info('DONE: NLP pipeline processed note files')
logger.log_info('Computing and outputting statistics')
line_list = []
for _ in self._loaded_df:
line_list.append(';')
table = self._do_statistics(_)
for _r in range(len(table[0])):
elems = [_c[_r] for _c in table]
line_list.append(';'.join(elems))
line_list.append(';')
line_list.append(';')
logger.log_info('CSV Table Output:')
for _ in line_list:
print(_)
def _query_bigquery(self):
sql_search = ""
merged_lexicon_map = {
_k: self._lexicon_map['positive'][_k] +
self._lexicon_map['negative'][_k]
for _k in self._lexicon_map['positive'].keys()
}
for _name, _terms in merged_lexicon_map.items():
if not _terms:
sql_search = sql_search + "," + " FALSE AS " + _name
else:
lex = [r'\\b' + x + r'\\b' for x in _terms]
sql_search = sql_search + "," + " REGEXP_CONTAINS(text, '(?i)(" + '|'.join(
lex) + ")') AS " + _name
ignore_str = '\n'.join([
'AND category NOT LIKE "%{}%"'.format(_)
for _ in self._ignore_cat_list
])
use_bqstorage_api = config.get_pipeline_config_item(
self.module_name(), "use_bqstorage_api", False)
limitstr = ""
if config.get_pipeline_config_item(self.module_name(),
"debug_download", False):
limitstr = 'LIMIT 10'
cohort_ids = []
if self._cohort_file and os.path.isfile(self._cohort_file):
cohort_ids = pd.read_csv(self._cohort_file)
cohort_ids.columns = [_.lower() for _ in cohort_ids.columns]
cohort_ids = list(cohort_ids.loc[:, 'hadm_id'])
sql = """
SELECT row_id, subject_id, hadm_id, chartdate, category, text{}
FROM `physionet-data.mimiciii_notes.noteevents`
WHERE hadm_id IS NOT NULL
AND hadm_id IN ({})
{}
{}
""".format(sql_search, ','.join([str(_) for _ in cohort_ids]),
ignore_str, limitstr)
logger.log_info('Querying noteevents for lexicon occurences.')
self._labeled_df = pandas_gbq.read_gbq(
sql,
project_id=google_tools.PROJECT_ID,
dialect='standard',
use_bqstorage_api=use_bqstorage_api
) #, progress_bar_type=utils.PROGRESSBAR_TYPE)
self._labeled_df.columns = [
_.upper() for _ in self._labeled_df.columns
]
if not self._dump_all:
mask = None
for _ in self._labeled_df.columns:
if _.lower() in [
'subject_id', 'row_id', 'hadm_id', 'chartdate',
'category', 'text'
]:
continue
if mask is None:
mask = self._labeled_df[_].astype(bool)
else:
mask = mask | self._labeled_df[_].astype(bool)
self._labeled_df = self._labeled_df[mask].copy()
logger.log_info('DONE: Querying noteevents for lexicon occurences.')
logger.log_debug('Number of admissions {}, number of notes {}.'.format(
self._labeled_df['HADM_ID'].nunique(), len(self._labeled_df)))
for _key in self._lexicon_map['positive'].keys():
_key = _key.upper()
logger.log_debug('Number of notes with {}: {}.'.format(
_key.lower(), self._labeled_df[_key.upper()].sum()))