def _get_features(self, ew, signal_to_truth_shift=0, valid=False, mask=np.ones((10), dtype=bool)): X = np.zeros((1, 10)) i = self.ew2i[ew] X[0, 0] = 1 for lag in range(3): if valid and not self.valid[i - lag][lag]: w = self.i2ew[i - lag] raise Exception('missing unstable wILI (ew=%d|lag=%d)' % (w, lag)) try: X[0, 1 + lag] = np.log( np.maximum( 0.01, self.data[i - lag - signal_to_truth_shift]['stable'])) except Exception: X[0, 1 + lag] = np.nan for holiday in range(4): if EW.split_epiweek(EW.add_epiweeks(ew, holiday))[1] == 1: X[0, 4 + holiday] = 1 y, w = EW.split_epiweek(ew) N = EW.get_num_weeks(y) offset = np.pi * 2 * w / N X[0, 8] = np.sin(offset) X[0, 9] = np.cos(offset) # todo linear time trend covariate? return X[:, mask]
def _get_features(self, ew, valid=True): X = np.zeros((1, 7)) i = self.ew2i[ew] X[0, 0] = 1 for holiday in range(4): if EW.split_epiweek(EW.add_epiweeks(ew, holiday))[1] == 1: X[0, 1 + holiday] = 1 y, w = EW.split_epiweek(ew) N = EW.get_num_weeks(y) offset = np.pi * 2 * w / N X[0, 5] = np.sin(offset) X[0, 6] = np.cos(offset) # todo linear time trend covariate? return X
def _get_partial_trajectory(self, epiweek, valid=True): y, w = EW.split_epiweek(epiweek) if w < 30: y -= 1 ew1 = EW.join_epiweek(y, 30) ew2 = epiweek limit = EW.add_epiweeks(ew2, -5) weeks = Epidata.range(ew1, ew2) stable = Epidata.check(Epidata.fluview(self.region, weeks)) try: unstable = Epidata.check(Epidata.fluview(self.region, weeks, issues=ew2)) except: unstable = [] wili = {} for row in stable: ew, value = row['epiweek'], row['wili'] if not valid or ew < limit: wili[ew] = value for row in unstable: ew, value = row['epiweek'], row['wili'] wili[ew] = value curve = [] for ew in EW.range_epiweeks(ew1, ew2, inclusive=True): if ew not in wili: if valid: t = 'unstable' else: t = 'any' raise Exception('wILI (%s) not available for week %d' % (t, ew)) curve.append(wili[ew]) n1 = EW.delta_epiweeks(ew1, ew2) + 1 n2 = len(curve) if n1 != n2: raise Exception('missing data (expected %d, found %d)' % (n1, n2)) return curve
def __init__(self, region): self.region = region weeks = Epidata.range(200330, 202330) rows = Epidata.check(Epidata.fluview(self.region, weeks)) self.seasons = {} for row in rows: ew, wili = row['epiweek'], row['wili'] y, w = EW.split_epiweek(ew) if w < 30: y -= 1 i = EW.delta_epiweeks(EW.join_epiweek(y, 30), ew) if y not in self.seasons: self.seasons[y] = {} if 0 <= i < 52: self.seasons[y][i] = wili years = sorted(list(self.seasons.keys())) for year in years: if len(self.seasons[year]) != 52: del self.seasons[year] if 2008 in self.seasons and 2009 in self.seasons: for i in range(40, 52): self.seasons[2008][i] = self.seasons[2009][i] del self.seasons[2009] curve = lambda y: [self.seasons[y][i] for i in range(52)] self.years = sorted(list(self.seasons.keys())) self.curves = dict([(y, curve(y)) for y in self.years])
def _ew2date(ew): # parse the epiweek year, week = flu.split_epiweek(ew) # get the date object (middle of the week; Wednesday) date = EpiDate.from_epiweek(year, week) # go to the first day of the week (Sunday) date = date.add_days(-3) # date as string return str(date)
def _get_features(self, ew, valid=True): X = np.zeros((1, 10)) i = self.ew2i[ew] X[0, 0] = 1 for lag in range(3): if valid and not self.valid[i - lag][lag]: w = self.i2ew[i - lag] raise Exception('missing unstable %s (ew=%d|lag=%d)' % (self.target, w, lag)) X[0, 1 + lag] = self.data[i - lag][lag] for holiday in range(4): if EW.split_epiweek(EW.add_epiweeks(ew, holiday))[1] == 1: X[0, 4 + holiday] = 1 y, w = EW.split_epiweek(ew) N = EW.get_num_weeks(y) offset = np.pi * 2 * w / N X[0, 8] = np.sin(offset) X[0, 9] = np.cos(offset) return X
def predict(self, epiweek, train=True, valid=True): if train: self.train(epiweek) if self.training_week > epiweek: raise Exception('trained on future data') y, w = EW.split_epiweek(epiweek) #if 30 <= w < 40: # return float(self.model.mean[w - 30]) #if 20 < w < 30: # return float(self.model.mean[-(30 - w)]) if 20 <= w < 39: raise Exception('no prediction on weeks 21--39') curve = self._get_partial_trajectory(epiweek, valid=valid) arch = self._fit(curve) return float(arch[len(curve)])
def _train(self, region): if region in self.bf_var: # already trained return if len(region) == 2: # TODO: this is a hack for state ILI # assume backfill of region 4 print('FIXME: setting backfill for %s as hhs4' % region) self.bf_var[region] = self.bf_var['hhs4'] self.emp_mean[region] = self.emp_mean['hhs4'] self.emp_var[region] = self.emp_var['hhs4'] self.emp_curves[region] = self.emp_curves['hhs4'] return stable = self._get_stable(region) start_weeks = [flu.get_season(ew)[0] for ew in stable.keys()] curves = [] seasons = set( [flu.split_epiweek(ew)[0] for ew in start_weeks if ew is not None]) for s in seasons: ew1 = flu.join_epiweek(s + 0, 40) if self.forecast_type == ForecastType.WILI: ew2 = flu.add_epiweeks(ew1, 37) else: ew2 = flu.add_epiweeks(ew1, 29) # print("stable: ", stable) # print("range_epiweeks: ", [i for i in flu.range_epiweeks(ew1, ew2)]) curve = [stable[ew] for ew in flu.range_epiweeks(ew1, ew2)] curves.append(curve) self.emp_mean[region] = np.mean(curves, axis=0) self.emp_var[region] = np.var(curves, axis=0, ddof=1) self.emp_curves[region] = curves if self.backfill_weeks is None: self.bf_var[region] = [0] else: self.bf_var[region] = [] for lag in range(self.backfill_weeks): unstable = self._get_unstable(region, lag) changes = [ stable[ew] - unstable[ew] for ew in stable.keys() & unstable.keys() ] if len(changes) < 2: raise Exception('not enough data') self.bf_var[region].append(np.var(changes, ddof=1)) print( ' %5s: %s' % (region, ' '.join(['%.3f' % (b**0.5) for b in self.bf_var[region]])))
def extract_epiweek_and_team(filename): """ Extract the submission epiweek (epiweek of most recently published report) and the team name from the file name of a flu contest submission. The return value is a tuple of: 1. the submission epiweek (e.g. 201751) 2. the team name (e.g. "delphi-epicast") """ # this is the naming convention for 2017 flu contest submissions pattern = re.compile('^EW(\\d{2})-(.*)-(\\d{4})-(\\d{2})-(\\d{2}).csv$') match = pattern.match(os.path.basename(filename)) if match is None: # only able to parse this specific naming convention raise Exception() week = int(match.group(1)) team = match.group(2) year = int(match.group(3)) month = int(match.group(4)) day = int(match.group(5)) epiweek = EpiDate(year, month, day).get_ew() # We know the week number, but the year has to be inferred from the # submission date. Since the week of submission is never less than the week # of the most recent report, we can step backwards from the week of # submission until we find the expected week number. Ordinarily, this will # take exactly two steps. For example, data collected on 2017w51 is # reported on 2017w52, and our forecast is submitted on 2018w01; so we # start with 2018w01 and step backwards until find the first week 51, which # is 2017w51. if not 1 <= week <= 53: # prevent an infinite loop raise Exception('invalid week number: %d' % week) while Epiweek.split_epiweek(epiweek)[1] != week: epiweek = Epiweek.add_epiweeks(epiweek, -1) return epiweek, team
def cum_to_week(data): epiweeks = list(data.keys()) all_epiweeks = list( EW.range_epiweeks(min(epiweeks), max(epiweeks), inclusive=True)) result = np.zeros((len(all_epiweeks))) last_valid = (-1, 0) # (idx, value) for i in range(len(result)): ew = all_epiweeks[i] if ew in data: if data[all_epiweeks[i]] is not None: result[last_valid[0] + 1:i + 1] = (data[ew] - last_valid[1]) / float( i - last_valid[0]) # Evenly distribute missing counts last_valid = (i, data[ew]) yr, wk = EW.split_epiweek(all_epiweeks[i]) if EW.get_num_weeks(yr) == wk: result[ last_valid[0] + 1:i + 1] = 0 # Fill rest of year with 0s, not getting this information last_valid = (i, 0) # Start new year at 0 return {all_epiweeks[i]: result[i] for i in range(len(all_epiweeks))}
def load_csv(filename): timestamp = None epiweek, team = ForecastIO.extract_epiweek_and_team(filename) season_start = Epiweek.get_season(epiweek)[0] season = Epiweek.split_epiweek(season_start)[0] forecast = Forecast(season, timestamp, team, epiweek) # the default column layout # can be updated based on header row canonical_fields = [f.lower() for f in ( 'Location', 'Target', 'Type', 'Unit', 'Bin_start_incl', 'Bin_end_notincl', 'Value' )] field_to_column = dict(zip(canonical_fields, range(len(canonical_fields)))) # read the csv one row at a time with open(filename, 'r', newline='') as f: reader = csv.reader(f) for row in reader: # skip header row(s) fields = [f.lower() for f in row] if 'location' in fields: # update the field-to-column-index mapping field_to_column = dict(zip(fields, range(len(fields)))) continue # extract values values = [row[field_to_column[f]] for f in canonical_fields] location, target, type_, unit, start, end, value = values # update forecast ForecastIO.__load_row(forecast, location, target, type_, start, value) # return the group of forecasts per location return forecast
def load_submission(file, system=None, epiweek=None, insane=False, test=False, verbose=False): # logging log = print if verbose else lambda x: x # load the forecast log('loading %s...' % file) fc = ForecastIO.load_csv(file) log(' forecast file parsed') # set the system (team) name if system is None: systems = [ ('ec', 'delphi-epicast'), ('eb', 'delphi-eb'), ('sp', 'delphi-spline'), ('st', 'delphi-stat'), ('af', 'delphi-archefilter'), ] num_found = 0 for (name, label) in systems: if label in fc.team.lower(): num_found += 1 system = name if num_found != 1: raise Exception('unrecognized system/team name') log(' system: %s' % system) # override epiweek if epiweek is not None and flu.check_epiweek(epiweek): fc.epiweek = epiweek fc.epiweek2 = flu.split_epiweek(epiweek)[1] epiweek = fc.epiweek log(' epiweek: %d' % epiweek) # sanity check if insane: log(' sanity check skipped') else: fc.sanity_check() log(' sanity check passed') # export JSON string fc_json = ForecastIO.export_json_delphi(fc) log(' forecast exported (%.1f KB)' % (len(fc_json) / 1024)) # store forecast u, p = secrets.db.epi cnx = mysql.connector.connect(user=u, password=p, database='epidata') log(' connected to database') sql = """ INSERT INTO `forecasts` (`system`, `epiweek`, `json`) VALUES (%s, %s, %s) ON DUPLICATE KEY UPDATE `json` = %s """ values = (system, epiweek, fc_json, fc_json) if test: log(' test mode, no commit') else: cur = cnx.cursor() cur.execute(sql, values) cur.close() cnx.commit() log(' forecast committed') # cleanup cnx.close() log('forecast loaded')
def get_season(epiweek): """Return the first year of the season that contains the given epiweek.""" year, week = split_epiweek(epiweek) if week < 40: year -= 1 return year
def plot(forecasts, prefix, fig_label=''): # timing epiweek = forecasts[0][0].epiweek ew0, ew1 = flu.get_season(epiweek) num_weeks = flu.delta_epiweeks(ew0, ew1) + 1 year = flu.split_epiweek(ew0)[0] # plot settings x_ticks = [i for i in range(0, num_weeks, 3)] x_tick_labels = [ '%02d' % ForecastIO.get_index_week(i) for i in x_ticks ] y_ticks = [i for i in range(0, 14, 2)] regions = ['nat'] + ['hhs%s' % i for i in range(1, 11)] # TODO: avoid hardcoding these values everywhere baseline_values_2019 = [ 2.4, 1.9, 3.2, 1.9, 2.4, 1.9, 3.8, 1.7, 2.7, 2.4, 1.5 ] baselines = dict( (r, v) for (r, v) in zip(regions, baseline_values_2019)) bin_size = forecasts[0][0].ili_bin_size # get the somewhat sorted list of all unique locations locations = [] for info in forecasts: fc = info[0] for loc in fc.get_locations(): if loc not in locations: locations.append(loc) # plot each region for region in locations: # only consider forecasts that include this location region_forecasts = [] for info in forecasts: if info[0].has_forecast(region): region_forecasts.append(info) # center subplot plt.figure(figsize=(12, 12)) ax2 = plt.subplot(3, 2, 3) if region in baselines: plt.axhline(baselines[region], color='#888888') weeks = [i for i in range(flu.delta_epiweeks(ew0, epiweek) + 1)] values = Plotter.get_unstable_wILI(region, ew0, epiweek) plt.plot(weeks, values, color='#000000', linewidth=2) weeks = [flu.delta_epiweeks(ew0, epiweek) + i for i in range(1, 5)] for (forecast, label, color) in region_forecasts: fc = forecast.get_forecast(region) values = [fc.get_lookahead(i)['point'] for i in range(1, 5)] plt.plot(weeks, values, color=color, linewidth=2) ax2.set_xbound(0, 33) ax2.set_ybound(0, 12) ax2.set_xticks(x_ticks) ax2.set_yticks(y_ticks) ax2.set_xticklabels(x_tick_labels) ax2.get_xaxis().set_tick_params(labelbottom='on', labeltop='on') ax2.get_yaxis().set_tick_params(labelleft='on', labelright='on') # top subplot: peakweek top = Plotter.weekly_subplot(region_forecasts, region, plt.subplot(3, 2, 1), ax2, False) # bottom subplot: onset bottom = Plotter.weekly_subplot(region_forecasts, region, plt.subplot(3, 2, 5), ax2, True) # right subplot: peakheight right = Plotter.wili_subplot(region_forecasts, region, plt.subplot(3, 2, 4), ax2, bin_size) # top-right subplot: legend leg = plt.subplot(3, 2, 2) for (forecast, label, color) in forecasts: plt.plot([0], [0], color=color, label=label) plt.legend(loc='lower left') # other stuff top.set_ylabel('Pr(Peak Week)') top.get_yaxis().set_label_position('right') bottom.set_ylabel('Pr(Onset Week)') bottom.get_yaxis().set_label_position('right') right.set_xlabel('Pr(Peak Height)') right.get_xaxis().set_label_position('top') ax2.set_ylabel('%s %s' % (fig_label, region.upper())) ax2.get_yaxis().set_label_position('left') # show the finished figure if prefix is None: plt.show() break else: filename = '%s_%s.png' % (prefix, region) plt.savefig(filename, bbox_inches='tight') print('saved %s' % filename)
def forecast(self, epiweek): """ `epiweek`: the most recent epiweek for which ILINet data is available """ # sanity checks flu.check_epiweek(epiweek) season = flu.split_epiweek(flu.get_season(epiweek)[0])[0] week = flu.split_epiweek(epiweek)[1] first_epiweek = flu.join_epiweek(season, 40) offset = flu.delta_epiweeks(first_epiweek, epiweek) if season != self.test_season: raise Exception('unable to forecast season %d' % season) if 20 < week < 40: raise Exception('unable to forecast week %02d' % week) # initialize forecast forecast = Forecast(self.test_season, datetime.now(), self.name, epiweek, self.forecast_type) # aliases for readability num_week_bins = forecast.season_length num_wili_bins = forecast.num_ili_bins wili_bin_size = forecast.ili_bin_size # if (forecast_type == ForecastType.HOSP): # num_wili_bins = 601 # uniform blending weights week_weight = self.min_week_prob * (num_week_bins + 1 ) # include `none` "bin" wili_weight = self.min_wili_prob * num_wili_bins if week_weight > 1: raise Exception('`min_week_prob` is impossibly high') if wili_weight > 1: raise Exception('`min_wili_prob` is impossibly high') # forecast each region for region in self.locations: # draw sample curves curves = self._forecast(region, epiweek) # regional info if Locations.is_region(region): baseline = Targets.baselines[self.test_season][region] else: baseline = None # get all targets targets = [ Targets.get_all_targets(c, baseline, offset, rule_season=self.test_season) for c in curves ] onsets = [t['onset'] for t in targets] peakweeks = [t['peakweek'] for t in targets] peaks = [t['peak'] for t in targets] x1s = [t['x1'] for t in targets] x2s = [t['x2'] for t in targets] x3s = [t['x3'] for t in targets] x4s = [t['x4'] for t in targets] # forecast each target allow_no_pw = self.test_season < 2016 if Locations.is_region(region): # skip onset for states and hospitalization, and do it only for regions onset = self.forecast_weeks(first_epiweek, num_week_bins, onsets, week_weight, self.smooth_weeks_bw, True) peakweek = self.forecast_weeks(first_epiweek, num_week_bins, peakweeks, week_weight, self.smooth_weeks_bw, allow_no_pw) peak = self.forecast_wili(wili_bin_size, num_wili_bins, peaks, wili_weight, self.smooth_wili_bw) x1 = self.forecast_wili(wili_bin_size, num_wili_bins, x1s, wili_weight, self.smooth_wili_bw) x2 = self.forecast_wili(wili_bin_size, num_wili_bins, x2s, wili_weight, self.smooth_wili_bw) x3 = self.forecast_wili(wili_bin_size, num_wili_bins, x3s, wili_weight, self.smooth_wili_bw) x4 = self.forecast_wili(wili_bin_size, num_wili_bins, x4s, wili_weight, self.smooth_wili_bw) # fill in the forecast data fc = forecast.get_or_create_forecast(region) if Locations.is_region(region): fc.set_onset(*onset) fc.set_peakweek(*peakweek) fc.set_peak(*peak) fc.set_lookahead(1, *x1) fc.set_lookahead(2, *x2) fc.set_lookahead(3, *x3) fc.set_lookahead(4, *x4) # sanity check completed forecast forecast.sanity_check() return forecast