def _completion_handler(self, status, message=''):
"""
Completion callback.
:type status: int
:param status:
:return:
"""
self.running = False
if not self._want_quit:
style = ''
status_message = ''
if status == workerthread.CompletionStatus.ABORT:
status_message = 'Computation successfully aborted. '
elif status == workerthread.CompletionStatus.DONE:
open_folder(self.output_folder_path)
status_message = 'Processing complete. '
elif status == workerthread.CompletionStatus.FAIL:
open_folder(self.output_folder_path)
status_message = 'Processing failed. '
style = 'error'
status_notifier.update({
'progress': (int(not status), 1),
'sub_progress': (int(not status), 1)
})
self.action_button.Enable(True)
self.action_button.SetLabel('Start')
self.enable_ui(True)
status_logger.info(status_message)
self.StatusBar.SetStatusText('')
with self._completion_lock:
self._completion_lock.notifyAll()
def run(self):
super(RulesPrep, self).run()
status_logger.info('{} :: Processing rules data'.format(
self.AGE_GROUP.capitalize()))
status_notifier.update({'progress': 1})
headers, matrix = DataPrep.read_input_file(self.input_file_path())
headers.extend(ADDITIONAL_DATA.keys())
status_notifier.update({'sub_progress': (0, len(matrix))})
for index, row in enumerate(matrix):
self.check_abort()
status_notifier.update({'sub_progress': (index, )})
self.expand_row(row, ADDITIONAL_DATA)
for rule in self.rules:
try:
if rule.logic_rule(row) is True:
row[RULES_CAUSE_NUM_KEY] = rule.CAUSE_ID
break
except Exception as e:
warning_logger.warning(
'SID: {} rule `{}` failed complete: {}'.format(
row['sid'], rule, e.message))
status_notifier.update({'sub_progress': None})
DataPrep.write_output_file(headers, matrix, self.output_file_path())
return matrix
def _read_graph_data(self):
super(CSMFGrapher, self)._read_graph_data()
# build ordered dict for values to be graphed. indexed by module
graph_data_unsorted = defaultdict(dict)
status_notifier.update({'sub_progress': (0, len(MODULE_LABELS))})
for cnt, module_key in enumerate(MODULE_LABELS):
status_notifier.update({'sub_progress': (cnt, )})
try:
with open(
os.path.join(
self.input_dir_path,
INPUT_FILENAME_TEMPLATE.format(module_key)),
'rb') as f:
reader = csv.DictReader(f)
for row in reader:
self.check_abort()
cause_key = row['cause'].rstrip()
cause_fraction = row['CSMF']
graph_data_unsorted[module_key][cause_key] = float(
cause_fraction)
except IOError:
# The file isn't there, there was no data or an error, so just skip it
continue
for sex in ('male', 'female'):
try:
key = '-'.join([module_key, sex])
filename = os.path.join(self.input_dir_path,
'{:s}-csmf.csv'.format(key))
with open(filename, 'rb') as f:
for row in csv.DictReader(f):
self.check_abort()
cause_key = row['cause'].rstrip()
cause_fraction = row['CSMF']
graph_data_unsorted[key][cause_key] = float(
cause_fraction)
except IOError:
continue
return graph_data_unsorted
def generate_cause_rankings(self, scored, uniform_scores):
"""Determine rank for each cause.
The scored user data is ranked against the scores from the validation
data which has been resampled to a uniform cause distribtuion. If an
observation is scored higher than any observation in the training data
it is ranked 0.5. If an observation is scored lower than any
observation in the training data it is ranked len(training) + 0.5.
The user_data is modified in place and not returned.
Args:
scored (list): list of ScoredVAs from user data
uniform_scores (dict of lists): sorted distribution of scores
by cause from uniform training data
"""
status_notifier.update({'sub_progress': (0, len(scored))})
for index, va in enumerate(scored):
status_notifier.update({'sub_progress': (index, )})
for cause in self.cause_list:
self.check_abort()
gt = bisect_left(uniform_scores[cause], va.scores[cause])
lt = bisect_right(uniform_scores[cause], va.scores[cause])
avg_rank = len(uniform_scores[cause]) - gt + (gt - lt) / 2.
va.ranks[cause] = avg_rank + .5
status_notifier.update({'sub_progress': None})
def score_symptom_data(self, symptom_data, tariffs):
"""Score symptom data using a tariffs matrix.
Args:
symptom_data (list of dict): symptom data from a csv.DictReader
tariffs (dict of lists): processed tariffs by cause
Returns:
list: List of Scored VAs.
"""
scored = []
status_notifier.update({'sub_progress': (0, len(symptom_data))})
for index, row in enumerate(symptom_data):
self.check_abort()
status_notifier.update({'sub_progress': (index, )})
va = self.score_row(row, tariffs)
va.censored = map(safe_int, row.get('restricted', '').split())
va.rules = safe_int(row.get(RULES_CAUSE_NUM_KEY))
scored.append(va)
status_notifier.update({'sub_progress': None})
return scored
def _read_graph_data(self):
graph_data = defaultdict(get_default_dict)
status_notifier.update({'sub_progress': (0, len(MODULE_LABELS))})
for cnt, module_key in enumerate(MODULE_LABELS):
status_notifier.update({'sub_progress': (cnt, )})
try:
with open(
os.path.join(
self.input_dir_path,
INPUT_FILENAME_TEMPLATE.format(module_key)),
'rb') as f:
reader = csv.DictReader(f)
for row in reader:
self.check_abort()
try:
age_key = get_age_key(float(row['age']))
if age_key not in AGE_DATA.values():
raise ValueError('Unknown age group.')
sex_key = int(row['sex'])
if sex_key not in [1, 2]:
raise ValueError(
'Cannot yet plot when sex is not M/F')
except ValueError as e:
# Age or sex is invalid. Log warning and skip this item.
warning_logger.warning(
'Cause Grapher :: SID {} value for age or sex is invalid.'
.format(row['sid'], e.message))
continue
graph_data[row['cause34']][sex_key][age_key] += 1
graph_data['All'][sex_key][age_key] += 1
except IOError:
# The file isn't there, there was no data or an error, so just skip it.
continue
return graph_data
def run(self):
super(WHOPrep, self).run()
status_logger.info('Mapping WHO Questionnaire')
status_notifier.update({'progress': 1})
headers = set(self.data_module.ADDITIONAL_HEADERS)
headers.update(self.data_module.YES_NO_QUESTIONS)
headers.update([h for h, _ in self.data_module.RECODE_QUESTIONS])
headers.update(self.data_module.RENAME_QUESTIONS)
headers.update(self.data_module.REVERSE_ONE_HOT_MULTISELECT)
headers.update([h for h, _ in self.data_module.RECODE_MULTISELECT])
headers.update(self.data_module.ONE_HOT_FROM_MULTISELECT)
headers.update(self.data_module.UNIT_IF_AMOUNT)
for unit_col, value_col, _ in self.data_module.DURATION_CONVERSIONS:
headers.update([unit_col, value_col])
_, matrix = DataPrep.read_input_file(self.input_file_path())
status_notifier.update({'sub_progress': (0, len(matrix))})
for index, row in enumerate(matrix):
self.check_abort()
status_notifier.update({'sub_progress': (index,)})
self.determine_consent(row)
self.calculate_age(row)
self.recode_yes_no_questions(row)
self.recode_categoricals(row)
self.rename_questions(row)
self.reverse_one_hot_multiselect(row)
self.recode_multiselects(row)
self.encode_one_hot_from_multiselect(row)
self.map_units_from_values(row)
self.convert_durations(row)
self.map_adult_chest_pain_duration(row)
self.map_child_illness_duration(row)
self.map_neonate_first_cry(row)
self.map_child_unconsciousness_start(row)
self.map_neonate_delivery_type(row)
self.map_child_birth_size(row)
self.map_redundant_child_age_data(row)
status_notifier.update({'sub_progress': None})
DataPrep.write_output_file(sorted(headers), matrix,
self.output_file_path(None))
def _make_graphs(self, graph_data):
# Make cause of death graphs.
status_notifier.update({'sub_progress': (0, len(graph_data))})
for cnt, (cause_key, data) in enumerate(graph_data.items()):
self.check_abort()
status_notifier.update({'sub_progress': (cnt, )})
make_graph(data, cause_key, self.output_dir_path)
status_notifier.update({'sub_progress': None})
def _make_graphs(self, graph_data_unsorted):
super(CSMFGrapher, self)._make_graphs(graph_data_unsorted)
# Make csmf graphs.
status_notifier.update({'sub_progress': (0, len(graph_data_unsorted))})
for cnt, (module_key, data) in enumerate(graph_data_unsorted.items()):
self.check_abort()
status_notifier.update({'sub_progress': (cnt, )})
# sort data in decreasing order
graph_data = OrderedDict(
sorted(data.iteritems(), key=lambda x: x[1], reverse=True))
make_graph(graph_data, module_key, self.output_dir_path)
status_notifier.update({'sub_progress': None})
def _update_status(self):
status_logger.info('Making cause graphs')
status_notifier.update({'progress': 1})
def _update_status(self):
super(CSMFGrapher, self)._update_status()
status_logger.info('Making CSMF graphs')
status_notifier.update({'progress': 1})
def process_training_data(self, train, tariffs, frequencies, cutoff_pos,
thresholds):
"""Process the training data.
The validated data is expanded so that the cause distribution across
all the observation is uniformly distributed across the causes. The
sampling frequencies are determined elsewhere and stored in the data
module.
Cause-specific cutoffs are calculated as the rank value at the given
cutoff percentile of the subset of observations whose gold standards
is the given cause. While the data are sorted also store the
distribution of scores by cause. This is used to rank the user data.
Args:
train (list): List of validated ScoredVAs.
frequencies (dict): Map of validated sid to frequency.
cutoff_pos (float): Percentile cutoff from 0 to 1.
Returns:
tuple:
list: validated VAs with uniform cause distribution.
dict: ordered scores by cause for all VAs in the training data
dict: mapping of ranks of VAs in the training data for which
the gold standard is the given cause
dict: Cutoff score for each cause.
"""
uniform_train = []
status_notifier.update({'sub_progress': (0, 1)})
for index, row in enumerate(train):
self.check_abort()
# Assume half the processing time is scoring/expanding
# Fill half the status bar based on the number of rows
status_notifier.update(
{'sub_progress': ((index / 2) / len(train), )})
va = self.score_row(row, tariffs)
va.cause = row.get('va46')
uniform_train.extend([va] * frequencies[va.sid])
scores = {}
ranks = {}
cutoffs = {}
likelihoods = {}
n_causes = len(self.cause_list)
n_uniform = len(uniform_train)
overall_cutoff = n_uniform * self.data_module.CUTOFF_POS
for index, cause in enumerate(self.cause_list):
self.check_abort()
# Assume half the processing time is sorting/ranking
# Start at 50% and updated in even increments for each cause
status_notifier.update(
{'sub_progress': (.5 + (index / 2) / n_causes, )})
# Get the uniform training data sorted by (reversed) score and
# sid. Sorting by sid ensures the ranks are stable between row
# which have the score but different gold standard causes.
def sorter(va):
return -va.scores[cause], va.sid
uniform_sorted = sorted(uniform_train, key=sorter)
# Store the scores from the distribution sorted from low to high
scores[cause] = [va.scores[cause] for va in uniform_sorted][::-1]
# Determine the rank within the complete uniform training data
# of the subset of VAs whose gold standard cause is the cause
# by which the VAs are ranked.
ranks = [(i + 1) for i, va in enumerate(uniform_sorted)
if int(va.cause) == cause]
n_ranks = len(ranks)
ranks[cause] = ranks
# Find the index of the item at cutoff position.
cutoffs[cause] = ranks[int(n_ranks * cutoff_pos)]
# Find the rank value at each threshold value
like = [0]
like.extend([ranks[int(n_ranks * thre)] for thre in thresholds])
like.append(min([cutoffs[cause], overall_cutoff]))
like.append(n_uniform)
likelihoods[cause] = like
status_notifier.update({'sub_progress': None})
return uniform_train, scores, ranks, cutoffs, likelihoods
def run(self):
super(TariffPrep, self).run()
status_logger.info('{:s} :: Processing tariffs'.format(
self.AGE_GROUP.capitalize()))
status_notifier.update({'progress': 1})
# Headers are being dropped only from tariff matrix now because of the
# way we are iterating over the pruned tariff data. It is unnecessary
# to drop headers from other matrices.
drop_headers = {TARIFF_CAUSE_NUM_KEY}
if not self.hce:
drop_headers.update(self.data_module.HCE_DROP_LIST)
if not self.free_text:
drop_headers.update(self.data_module.FREE_TEXT)
if self.short_form:
drop_headers.update(self.data_module.SHORT_FORM_DROP_LIST)
tariffs = get_tariff_matrix(self.tariffs_filename, drop_headers,
self.data_module.SPURIOUS_ASSOCIATIONS)
self.cause_list = sorted(tariffs.keys())
validated = self.read_input_file(self.validated_filename)[1]
status_logger.info('{:s} :: Processing validation data.'.format(
self.AGE_GROUP.capitalize()))
train = self.process_training_data(validated, tariffs,
self.data_module.FREQUENCIES,
self.data_module.CUTOFF_POS,
[.25, .5, .75])
(uniform_train, uniform_scores, uniform_ranks, cutoffs,
likelihoods) = train
self.write_cutoffs(cutoffs)
status_logger.info('{:s} :: Generating VA cause list.'.format(
self.AGE_GROUP.capitalize()))
user_data = self.read_input_file(self.input_file_path())[1]
user_data = self.score_symptom_data(user_data, tariffs)
status_logger.info('{:s} :: Generating cause rankings.'.format(
self.AGE_GROUP.capitalize()))
self.generate_cause_rankings(user_data, uniform_scores)
self.write_intermediate_file(user_data, 'external-ranks', 'ranks')
lowest_rank = len(uniform_train) + 0.5
self.mask_ranks(user_data, len(uniform_train), cutoffs,
self.data_module.CAUSE_CONDITIONS, lowest_rank,
self.data_module.UNIFORM_LIST_POS,
self.data_module.MIN_CAUSE_SCORE)
self.predict(user_data, lowest_rank, self.data_module.CAUSE_REDUCTION,
self.data_module.CAUSES, self.data_module.CAUSES46)
self.determine_likelihood(user_data, likelihoods,
self.data_module.CAUSE_REDUCTION)
undetermined_weights = self._get_undetermined_matrix()
csmf, csmf_by_sex = self.calculate_csmf(user_data,
undetermined_weights)
self.write_predictions(user_data)
likelihood_names = [
'Very Likely', 'Likely', 'Somewhat Likely', 'Possible'
]
if self.language != 'english':
path = os.path.join(config.basedir, 'data',
'{}.json'.format(self.language))
with open(path, 'rb') as f:
translation = json.load(f)
likelihood_names = [
translation['likelihoods'].get(likelihood)
for likelihood in likelihood_names
]
else:
translation = None
colors = ['#3CB371', '#47d147', '#8ae600', '#e6e600']
mp = self.write_multiple_predictions_xlsx(user_data, tariffs,
likelihood_names, colors,
translation)
self.write_multiple_predictions_csv(mp)
self.write_csmf(self.AGE_GROUP, csmf)
sex_name = {1: 'male', 2: 'female'}
for sex, csmf_data in csmf_by_sex.items():
key = '-'.join([self.AGE_GROUP, sex_name[sex]])
self.write_csmf(key, csmf_data)
self.write_intermediate_file(user_data, 'tariff-scores', 'scores')
self.write_intermediate_file(user_data, 'tariff-ranks', 'ranks')
return user_data
def run(self):
status_logger.info('Preparing variable headers.')
status_notifier.update({'progress': (0, 15), 'sub_progress': None})
intermediate_dir = intermediate_dir_path(self.output_dir_path)
figures_dir = os.path.join(self.output_dir_path, 'figures')
self.make_dir(intermediate_dir_path(self.output_dir_path))
try:
self.format_headers(self.input_file_path, os.path.join(intermediate_dir, CLEAN_HEADERS_FILENAME))
except StopIteration:
# File doesn't contain data
message = 'Source file "{}" does not contain data.'.format(self.input_file_path)
self._complete(CompletionStatus.FAIL, message)
warning_logger.warning(message)
return
report_logger.info('Analysis parameters:')
report_logger.info('- Input file: {}'.format(self.input_file_path))
report_logger.info('- Output folder: {}'.format(self.output_dir_path))
report_logger.info('- Country: {}'.format(self.country))
report_logger.info('- HIV Region: {}'.format(self.options.get('hiv', True)))
report_logger.info('- Malaria Region: {}'.format(self.options.get('malaria', True)))
report_logger.info('')
file_path = os.path.join(intermediate_dir, CLEAN_HEADERS_FILENAME)
who_questionnaire = self.who_questionaire_test(file_path)
if who_questionnaire:
self.short_form = True
form_name = 'WHO 2016 Questionnaire'
else:
self.short_form = self.short_form_test(file_path)
warning_logger.debug('Detected {} form'.format(
'short' if self.short_form else 'standard'))
if self.short_form:
form_name = 'PHMRC Shortened Questionnaire'
else:
form_name = 'PHMRC Full Questionnaire'
report_logger.info('Detected {}'.format(form_name))
who_prep = WHOPrep(self.output_dir_path)
common_prep = CommonPrep(self.output_dir_path, self.short_form)
adult_pre_symptom = PreSymptomPrep(adult_pre_symptom_data, self.output_dir_path, self.short_form)
adult_rules = RulesPrep(self.output_dir_path, self.short_form, common_data.ADULT, ADULT_RULES)
adult_symptom = SymptomPrep(adult_symptom_data, self.output_dir_path, self.short_form)
adult_results = TariffPrep(adult_tariff_data, self.output_dir_path, self.short_form, self.options, self.country)
child_pre_symptom = PreSymptomPrep(child_pre_symptom_data, self.output_dir_path, self.short_form)
child_rules = RulesPrep(self.output_dir_path, self.short_form, common_data.CHILD, CHILD_RULES)
child_symptom = SymptomPrep(child_symptom_data, self.output_dir_path, self.short_form)
child_results = TariffPrep(child_tariff_data, self.output_dir_path, self.short_form, self.options, self.country)
neonate_pre_symptom = PreSymptomPrep(neonate_pre_symptom_data, self.output_dir_path, self.short_form)
neonate_rules = RulesPrep(self.output_dir_path, self.short_form, common_data.NEONATE, NEONATE_RULES)
neonate_symptom = SymptomPrep(neonate_symptom_data, self.output_dir_path, self.short_form)
neonate_results = TariffPrep(neonate_tariff_data, self.output_dir_path, self.short_form, self.options, self.country)
legacy = self.options.get('legacy_format', False)
output = OutputPrep(self.output_dir_path, reorganize=not legacy,
keep_orig=legacy, short_form=self.short_form,
free_text=self.options.get('free_text', True),
hce=self.options.get('hce', True))
cause_grapher = CauseGrapher(self.output_dir_path)
csmf_grapher = CSMFGrapher(self.output_dir_path)
self._abort_list.extend([
who_prep,
common_prep,
adult_pre_symptom,
adult_rules,
adult_symptom,
adult_results,
child_pre_symptom,
child_rules,
child_symptom,
child_results,
neonate_pre_symptom,
neonate_rules,
neonate_symptom,
neonate_results,
cause_grapher,
csmf_grapher,
])
try:
if who_questionnaire:
who_prep.run()
# makes adult-prepped.csv, child-prepped.csv, neonate-prepped.csv
adult_data, child_data, neonate_data = common_prep.run()
if adult_data:
# makes adult-presymptom.csv
adult_pre_symptom.run()
# makes adult-logic-rules.csv
adult_rules.run()
# makes adult-symptom.csv
adult_symptom.run()
# creates adult output files
adult_results.run()
if child_data:
# makes child-presymptom.csv
child_pre_symptom.run()
# makes child-logic-rules.csv
child_rules.run()
# makes child-symptom.csv
child_symptom.run()
# creates child output files
child_results.run()
if neonate_data:
# makes neonate-presymptom.csv
neonate_pre_symptom.run()
# makes neonate-logic-rules.csv
neonate_rules.run()
# makes neonate-symptom.csv
neonate_symptom.run()
# creates neonate output files
neonate_results.run()
if self.options.get('figures') and (adult_data or child_data or neonate_data):
self.make_dir(figures_dir)
# generate all cause graphs
cause_grapher.run()
# generate all csmf graphs
csmf_grapher.run()
output.run()
except AbortException:
self._complete(CompletionStatus.ABORT)
except Exception:
traceback.print_exc()
self._complete(CompletionStatus.FAIL)
else:
self._complete(CompletionStatus.DONE)
def run(self):
super(SymptomPrep, self).run()
status_logger.info('{} :: Processing symptom data'.format(
self.AGE_GROUP.capitalize()))
status_notifier.update({'progress': 1})
headers, matrix = DataPrep.read_input_file(self.input_file_path())
status_notifier.update({'sub_progress': (0, len(matrix))})
additional_data = {}
additional_data.update(self.data_module.GENERATED_VARS_DATA)
additional_headers, additional_values = additional_headers_and_values(
headers, additional_data.items())
headers.extend(additional_headers)
self.rename_headers(headers, self.data_module.VAR_CONVERSION_MAP)
keep_list = [
header for header in headers
if re.match(self.data_module.KEEP_PATTERN, header)
]
drop_list = self.data_module.DROP_LIST
headers = sorted([
header for header in headers
if header in keep_list and header not in drop_list
],
key=lambda t: (t != 'sid', t[1].isdigit(), t))
for index, row in enumerate(matrix):
self.check_abort()
status_notifier.update({'sub_progress': (index, )})
self.expand_row(row,
dict(zip(additional_headers, additional_values)))
self.rename_vars(row, self.data_module.VAR_CONVERSION_MAP)
self.copy_variables(row, self.data_module.COPY_VARS)
# Compute age quartiles.
self.process_progressive_value_data(
row, self.data_module.AGE_QUARTILE_BINARY_VARS.items())
self.process_cutoff_data(
row, self.data_module.DURATION_CUTOFF_DATA.items())
self.process_injury_data(row, self.data_module.INJURY_VARS.items())
# Dichotomize!
self.process_binary_vars(
row, self.data_module.BINARY_CONVERSION_MAP.items())
# Ensure all binary variables actually ARE 0 or 1:
self.post_process_binary_variables(row,
self.data_module.BINARY_VARS)
self.censor_causes(row, self.data_module.CENSORED_MAP)
self.require_symptoms(row, self.data_module.REQUIRED_MAP)
status_notifier.update({'sub_progress': None})
DataPrep.write_output_file(headers, matrix, self.output_file_path())
return matrix
def run(self):
"""Perform initial processing step for preparing input data.
Returns:
tuple(bool): Tuple of bool values if VAs are present for Adult, Child, and Neonate.
"""
super(CommonPrep, self).run()
status_logger.info('Initial data prep')
status_notifier.update({'progress': 1})
headers, matrix = DataPrep.read_input_file(self.input_file_path())
status_notifier.update({'sub_progress': (0, len(matrix))})
# Extend the headers with additional headers and read the remaining data into the matrix
additional_data = {k: '' for k in ADDITIONAL_HEADERS}
if self.short_form:
additional_data.update(SHORT_FORM_ADDITIONAL_HEADERS_DATA)
additional_headers, additional_values = additional_headers_and_values(
headers, additional_data.items())
headers.extend(additional_headers)
if 'child_1_8a' not in headers:
headers.append('child_1_8a')
for index, row in enumerate(matrix):
self.check_abort()
status_notifier.update({'sub_progress': (index, )})
self.check_sids(row, index)
if not self.check_consent(row, CONSENT_HEADER, index):
warning_logger.info('SID: {} Refused consent.'.format(
row['sid']))
continue
self.expand_row(row,
dict(zip(additional_headers, additional_values)))
self.correct_missing_age(row)
try:
self.convert_cell_to_int(row, AGE_VARS.values())
except KeyError as e:
warning_logger.error('Missing age variable: {}'.format(
e.message))
missing_vars = [
var for var in AGE_VARS.values() if var not in headers
]
status_logger.info('Cannot process data without: {}'.format(
', '.join(missing_vars)))
status_notifier.update('abort')
continue
for header, mapping in BINARY_CONVERSION_MAP.items():
self.process_multiselect_vars(row, header, mapping)
for header in COUNT_DATA_HEADERS:
self.process_count_data(row, header)
self.convert_rash_data(row, RASH_DATA)
self.convert_weight_data(row, WEIGHT_CONVERSION_DATA)
self.convert_free_text(row, FREE_TEXT_VARS, WORD_SUBS)
self.save_row(row, index)
status_notifier.update({'sub_progress': None})
self.write_data(headers, self._matrix_data)
return bool(self._matrix_data[ADULT]), bool(
self._matrix_data[CHILD]), bool(self._matrix_data[NEONATE])
def run(self):
super(PreSymptomPrep, self).run()
status_logger.info('{} :: Processing pre-symptom data'.format(
self.AGE_GROUP.capitalize()))
status_notifier.update({'progress': 1})
# Create a list of duration variables, dropping specified variables if using the short form.
duration_vars = self.data_module.DURATION_VARS[:]
if self.short_form:
for var in self.data_module.DURATION_VARS_SHORT_FORM_DROP_LIST:
duration_vars.remove(var)
headers, matrix = DataPrep.read_input_file(self.input_file_path())
status_notifier.update({'sub_progress': (0, len(matrix))})
# Identify new headers and data to be included.
additional_data = {k: '' for k in self.data_module.DURATION_VARS}
duration_day_vars = getattr(self.data_module, 'DURATION_DAYS_VARS', [])
additional_data.update({k: '' for k in duration_day_vars})
additional_data.update(
{k: 0
for k in self.data_module.GENERATED_VARS_DATA})
additional_data.update(
{k: 0
for k in sorted(self.data_module.WORDS_TO_VARS.values())})
additional_headers, additional_values = additional_headers_and_values(
headers, additional_data.items())
headers.extend(additional_headers)
self.rename_headers(headers, self.data_module.VAR_CONVERSION_MAP)
# Make a list of headers to keep and to drop.
keep_list = [
header for header in headers
if re.match(self.data_module.KEEP_PATTERN, header)
]
drop_list = (['{}a'.format(header) for header in duration_vars] +
['{}b'.format(header) for header in duration_vars])
# Prune headers and sort by 'sid', then anything that doesn't contain a digit at pos 1, then general vars.
headers = sorted(
[
header for header in headers
if header in keep_list and header not in drop_list
],
key=lambda t:
(t != 'sid', t[1].isdigit(), not t.startswith('g'), t))
for index, row in enumerate(matrix):
self.check_abort()
status_notifier.update({'sub_progress': (index, )})
self.expand_row(row,
dict(zip(additional_headers, additional_values)))
self.rename_vars(row, self.data_module.VAR_CONVERSION_MAP)
self.verify_answers_for_row(row, RANGE_LIST)
self.fix_agedays(row)
self.calculate_age_at_death_value(row)
self.recode_answers(row, self.data_module.RECODE_MAP)
self.process_binary_vars(
row, self.data_module.BINARY_CONVERSION_MAP.items())
self.calculate_duration_vars(
row, duration_vars,
self.data_module.DURATION_VARS_SPECIAL_CASE)
self.validate_days_vars(row, duration_day_vars)
self.validate_weight_vars(row, self.data_module.WEIGHT_VARS)
self.validate_date_vars(row, self.data_module.DATE_VARS)
self.process_age_vars(row)
self.convert_free_text_vars(row, self.data_module.FREE_TEXT_VARS,
self.data_module.WORDS_TO_VARS)
if self.short_form:
word_list = [
v for k, v in
self.data_module.SHORT_FORM_FREE_TEXT_CONVERSION.items()
if value_or_default(row.get(k)) == 1
]
if word_list:
self.convert_free_text_words(
row, word_list, self.data_module.WORDS_TO_VARS)
self.fix_rash_length(row)
self.fix_rash_location(row)
self.process_weight_sd_vars(
row, getattr(self.data_module, 'EXAM_DATE_VARS', {}),
getattr(self.data_module, 'WEIGHT_SD_DATA', {}))
self.fill_missing_data(row, self.default_fill)
status_notifier.update({'sub_progress': None})
DataPrep.write_output_file(headers, matrix, self.output_file_path())
return matrix