def main():
# args and usage
parser = argparse.ArgumentParser()
parser.add_argument('location', action='store', type=str, default=None, help='location(s) (ex: all; US; TX; CA,LA,WY)')
parser.add_argument('term', action='store', type=str, default=None, help='term/query/topic (ex: all; /m/0cycc; "flu fever")')
parser.add_argument('--first', '-f', default=None, type=int, help='first epiweek override')
parser.add_argument('--last', '-l', default=None, type=int, help='last epiweek override')
args = parser.parse_args()
# sanity check
first, last = args.first, args.last
if first is not None:
flu.check_epiweek(first)
if last is not None:
flu.check_epiweek(last)
if first is not None and last is not None and first > last:
raise Exception('epiweeks in the wrong order')
# decide what to update
if args.location.lower() == 'all':
locations = LOCATIONS
else:
locations = args.location.upper().split(',')
if args.term.lower() == 'all':
terms = TERMS
else:
terms = [args.term]
# run the update
update(locations, terms, first, last)
def main():
# args and usage
parser = argparse.ArgumentParser()
parser.add_argument('--location',
action='store',
type=str,
default=None,
help='location(s) (ex: all; any of hhs1-10)')
parser.add_argument('--first',
'-f',
default=None,
type=int,
help='first epiweek override')
parser.add_argument('--last',
'-l',
default=None,
type=int,
help='last epiweek override')
parser.add_argument('--force_update',
'-u',
action='store_true',
help='force update db values')
parser.add_argument('--skip_email',
'-s',
action='store_true',
help='skip email downloading step')
args = parser.parse_args()
# sanity check
first, last, force_update, skip_email = args.first, args.last, args.force_update, args.skip_email
load_email = not skip_email
if first is not None:
flu.check_epiweek(first)
if last is not None:
flu.check_epiweek(last)
if first is not None and last is not None and first > last:
raise Exception('epiweeks in the wrong order')
# decide what to update
if args.location.lower() == 'all':
locations = LOCATIONS
else:
locations = args.location.lower().split(',')
# run the update
update(locations, first, last, force_update, load_email)
def _get_flu_data(html):
week_pattern = re.compile('^categories: \\[(.*)\\],$')
value_pattern = re.compile('^series: \\[(.*)\\],$')
data = {}
parsing_ili = True
for line in html.split('\n'):
line = line.strip()
match = week_pattern.match(line)
if match is not None:
weeks = [int(x[1:-1]) for x in match.group(1).split(',')]
for week in weeks:
check_epiweek(week)
if week not in data:
data[week] = {}
match = value_pattern.match(line)
if match is not None:
for item in match.group(1).split('},{'):
parts = item.replace('{',
'').replace('}',
'').strip().split(' ')
location = parts[1][1:-2]
def num(value):
if parsing_ili:
return float(value)
else:
if '.' in value:
raise Exception('expected type int for visits')
return int(value)
values = [num(x) for x in parts[3][1:-1].split(',')]
unit = 'ili' if parsing_ili else 'visits'
if len(weeks) != len(values):
raise Exception('len(weeks) != len(values)')
for week, value in zip(weeks, values):
if location not in data[week]:
data[week][location] = {}
data[week][location][unit] = value
parsing_ili = False
if len(data) == 0:
raise Exception('no data')
return data
def main():
# args and usage
parser = argparse.ArgumentParser()
parser.add_argument('--first', '-f', default=None, type=int, help='first epiweek override')
parser.add_argument('--last', '-l', default=None, type=int, help='last epiweek override')
parser.add_argument('--epiweek', '-w', default=None, type=int, help='epiweek override')
parser.add_argument('--test', '-t', default=False, action='store_true', help='dry run only')
args = parser.parse_args()
# sanity check
first, last, week = args.first, args.last, args.epiweek
for ew in [first, last, week]:
if ew is not None:
flu.check_epiweek(ew)
if first is not None and last is not None and first > last:
raise Exception('epiweeks in the wrong order')
if week is not None:
first = last = week
# extract the page hits for all states on the specified weeks
extract(first, last, args.test)
def validate_args(args):
"""Validate and return command line arguments."""
# check epiweek specification
first, last, week = args.first, args.last, args.epiweek
for ew in [first, last, week]:
if ew is not None:
flu.check_epiweek(ew)
if week is not None:
if first is not None or last is not None:
raise ValueError('`week` overrides `first` and `last`')
first = last = week
if first is not None and last is not None and first > last:
raise ValueError('`first` must not be greater than `last`')
# validate and extract name-location pairs
pair_regex = '[^-,]+-[^-,]+'
names_regex = '%s(,%s)*' % (pair_regex, pair_regex)
if not re.match(names_regex, args.names):
raise ValueError('invalid sensor specification')
return args.names, first, last, args.valid, args.test, args.target
def get_paho_row(row):
if row[0] == "\ufeffIncidence Rate (c)" and row != "\ufeffIncidence Rate (c),(SD/D) x100 (e),CFR (f),ID,Country or Subregion,Deaths,EW,Confirmed,Epidemiological Week (a),Pop (no usar),Serotype,Severe Dengue (d),Total of Dengue Cases (b),Year,Population x 1000".split(
","):
raise Exception('PAHO header row has changed')
if len(row) == 1 or row[0] == "Incidence Rate (c)":
# this is a header row
return None
try:
country = pycountry.countries.get(name=row[4]).alpha_2
except Exception:
try:
country = pycountry.countries.get(common_name=row[4]).alpha_2
except Exception:
try:
country = pycountry.countries.get(official_name=row[4]).alpha_2
except Exception:
return None
try:
check_epiweek(
safe_int(row[13]) * 100 + safe_int(row[8]),
safe_int(row[13]) * 100 + safe_int(row[6]))
except Exception:
return None
return {
'issue': safe_int(row[13]) * 100 + safe_int(row[6]),
'epiweek': safe_int(row[13]) * 100 + safe_int(row[8]),
'region': country,
'total_pop': safe_int(row[14]),
'serotype': row[10],
'num_dengue': safe_int(row[12]),
'incidence_rate': safe_float(row[0]),
'num_severe': safe_int(row[11]),
'num_deaths': safe_int(row[5]),
'severe_ratio': safe_float(row[1]),
'cfr': safe_float(row[2])
}
def load_submission(file, system=None, epiweek=None, insane=False, test=False, verbose=False):
# logging
log = print if verbose else lambda x: x
# load the forecast
log('loading %s...' % file)
fc = ForecastIO.load_csv(file)
log(' forecast file parsed')
# set the system (team) name
if system is None:
systems = [
('ec', 'delphi-epicast'),
('eb', 'delphi-eb'),
('sp', 'delphi-spline'),
('st', 'delphi-stat'),
('af', 'delphi-archefilter'),
]
num_found = 0
for (name, label) in systems:
if label in fc.team.lower():
num_found += 1
system = name
if num_found != 1:
raise Exception('unrecognized system/team name')
log(' system: %s' % system)
# override epiweek
if epiweek is not None and flu.check_epiweek(epiweek):
fc.epiweek = epiweek
fc.epiweek2 = flu.split_epiweek(epiweek)[1]
epiweek = fc.epiweek
log(' epiweek: %d' % epiweek)
# sanity check
if insane:
log(' sanity check skipped')
else:
fc.sanity_check()
log(' sanity check passed')
# export JSON string
fc_json = ForecastIO.export_json_delphi(fc)
log(' forecast exported (%.1f KB)' % (len(fc_json) / 1024))
# store forecast
u, p = secrets.db.epi
cnx = mysql.connector.connect(user=u, password=p, database='epidata')
log(' connected to database')
sql = """
INSERT INTO
`forecasts` (`system`, `epiweek`, `json`)
VALUES
(%s, %s, %s)
ON DUPLICATE KEY UPDATE
`json` = %s
"""
values = (system, epiweek, fc_json, fc_json)
if test:
log(' test mode, no commit')
else:
cur = cnx.cursor()
cur.execute(sql, values)
cur.close()
cnx.commit()
log(' forecast committed')
# cleanup
cnx.close()
log('forecast loaded')
cnx.close()
if __name__ == '__main__':
# args and usage
parser = argparse.ArgumentParser()
parser.add_argument('names', type=str, help='list of name-location pairs (location can be nat/hhs/cen/state or specific location labels)')
parser.add_argument('--first', '-f', default=None, type=int, help='first epiweek override')
parser.add_argument('--last', '-l', default=None, type=int, help='last epiweek override')
parser.add_argument('--epiweek', '-w', default=None, type=int, help='epiweek override')
parser.add_argument('--test', '-t', default=False, action='store_true', help='dry run only')
parser.add_argument('--valid', '-v', default=False, action='store_true', help='do not fall back to stable wILI; require unstable wILI')
args = parser.parse_args()
# sanity check
first, last, week = args.first, args.last, args.epiweek
for ew in [first, last, week]:
if ew is not None:
flu.check_epiweek(ew)
if first is not None and last is not None and first > last:
raise Exception('epiweeks in the wrong order')
if week is not None:
first = last = week
# extract name-location pairs
sensors = [pair.split('-') for pair in args.names.split(',')]
# update the requested sensors
print(sensors, first, last, args.valid, args.test, sep=" ")
update(sensors, first, last, args.valid, args.test)
def forecast(self, epiweek):
"""
`epiweek`: the most recent epiweek for which ILINet data is available
"""
# sanity checks
flu.check_epiweek(epiweek)
season = flu.split_epiweek(flu.get_season(epiweek)[0])[0]
week = flu.split_epiweek(epiweek)[1]
first_epiweek = flu.join_epiweek(season, 40)
offset = flu.delta_epiweeks(first_epiweek, epiweek)
if season != self.test_season:
raise Exception('unable to forecast season %d' % season)
if 20 < week < 40:
raise Exception('unable to forecast week %02d' % week)
# initialize forecast
forecast = Forecast(self.test_season, datetime.now(), self.name,
epiweek, self.forecast_type)
# aliases for readability
num_week_bins = forecast.season_length
num_wili_bins = forecast.num_ili_bins
wili_bin_size = forecast.ili_bin_size
# if (forecast_type == ForecastType.HOSP):
# num_wili_bins = 601
# uniform blending weights
week_weight = self.min_week_prob * (num_week_bins + 1
) # include `none` "bin"
wili_weight = self.min_wili_prob * num_wili_bins
if week_weight > 1:
raise Exception('`min_week_prob` is impossibly high')
if wili_weight > 1:
raise Exception('`min_wili_prob` is impossibly high')
# forecast each region
for region in self.locations:
# draw sample curves
curves = self._forecast(region, epiweek)
# regional info
if Locations.is_region(region):
baseline = Targets.baselines[self.test_season][region]
else:
baseline = None
# get all targets
targets = [
Targets.get_all_targets(c,
baseline,
offset,
rule_season=self.test_season)
for c in curves
]
onsets = [t['onset'] for t in targets]
peakweeks = [t['peakweek'] for t in targets]
peaks = [t['peak'] for t in targets]
x1s = [t['x1'] for t in targets]
x2s = [t['x2'] for t in targets]
x3s = [t['x3'] for t in targets]
x4s = [t['x4'] for t in targets]
# forecast each target
allow_no_pw = self.test_season < 2016
if Locations.is_region(region):
# skip onset for states and hospitalization, and do it only for regions
onset = self.forecast_weeks(first_epiweek, num_week_bins,
onsets, week_weight,
self.smooth_weeks_bw, True)
peakweek = self.forecast_weeks(first_epiweek, num_week_bins,
peakweeks, week_weight,
self.smooth_weeks_bw, allow_no_pw)
peak = self.forecast_wili(wili_bin_size, num_wili_bins, peaks,
wili_weight, self.smooth_wili_bw)
x1 = self.forecast_wili(wili_bin_size, num_wili_bins, x1s,
wili_weight, self.smooth_wili_bw)
x2 = self.forecast_wili(wili_bin_size, num_wili_bins, x2s,
wili_weight, self.smooth_wili_bw)
x3 = self.forecast_wili(wili_bin_size, num_wili_bins, x3s,
wili_weight, self.smooth_wili_bw)
x4 = self.forecast_wili(wili_bin_size, num_wili_bins, x4s,
wili_weight, self.smooth_wili_bw)
# fill in the forecast data
fc = forecast.get_or_create_forecast(region)
if Locations.is_region(region):
fc.set_onset(*onset)
fc.set_peakweek(*peakweek)
fc.set_peak(*peak)
fc.set_lookahead(1, *x1)
fc.set_lookahead(2, *x2)
fc.set_lookahead(3, *x3)
fc.set_lookahead(4, *x4)
# sanity check completed forecast
forecast.sanity_check()
return forecast
def main():
# args and usage
parser = argparse.ArgumentParser()
parser.add_argument('location',
action='store',
type=str,
default=None,
help='location(s) (ex: all; US; TX; CA,LA,WY)')
parser.add_argument(
'term',
action='store',
type=str,
default=None,
help='term/query/topic (ex: all; /m/0cycc; "flu fever")')
parser.add_argument('--first',
'-f',
default=None,
type=int,
help='first epiweek override')
parser.add_argument('--last',
'-l',
default=None,
type=int,
help='last epiweek override')
parser.add_argument('--country',
'-c',
default='US',
type=str,
help='location country (ex: US; BR)')
args = parser.parse_args()
# sanity check
first, last = args.first, args.last
if first is not None:
flu.check_epiweek(first)
if last is not None:
flu.check_epiweek(last)
if first is not None and last is not None and first > last:
raise Exception('epiweeks in the wrong order')
# decide what to update
if args.location.lower() == 'all':
locations = LOCATIONS
elif args.location.lower() == 'none':
locations = [NO_LOCATION_STR]
else:
locations = args.location.upper().split(',')
if args.term.lower() == 'all':
terms = TERMS
else:
terms = [args.term]
# country argument
# Check that country follows ISO 1366 Alpha-2 code.
# See https://www.iso.org/obp/ui/#search.
countries = args.country.upper().split(',')
if not all(map(lambda x: len(x) == 2, countries)):
raise Exception('country name must be two letters (ISO 1366 Alpha-2)')
# if length of locations and countries is > 1, need to be the same
if len(locations) > 1 and len(countries) > 1 and len(locations) != len(
countries):
raise Exception(
'locations and countries must be length 1, or same length')
# run the update
update(locations, terms, first, last, countries)
help='first epiweek override')
parser.add_argument('--last',
'-l',
default=None,
type=int,
help='last epiweek override')
parser.add_argument('--test',
'-t',
default=False,
action='store_true',
help='dry run only')
args = parser.parse_args()
# epiweeks and timing
first, last = None, None
if args.first is not None:
first = args.first
if args.last is not None:
last = args.last
if last is None:
last = get_most_recent_issue()
if first is None:
first = flu.add_epiweeks(last, -52)
if last < first:
raise Exception('epiweeks in the wrong order')
flu.check_epiweek(first, last)
print('Updating epiweeks from %d to %d.' % (first, last))
# make it happen
update(first, last, args.test)