def update():
now = get_timestr()
# get fhi datafile
datafile = get_fhi_datafile("data_covid19_lab_by_time")
df_new = pd.read_csv(datafile,
usecols=["date", "n_neg", "n_pos", "pr100_pos"],
parse_dates=["date"])
mapping = {"n_neg": "new_neg", "n_pos": "new_pos"}
df_new = df_new.rename(columns=mapping)
df_new["new_total"] = df_new["new_neg"] + df_new["new_pos"]
df_new["total_neg"] = df_new["new_neg"].cumsum()
df_new["total_pos"] = df_new["new_pos"].cumsum()
df_new["total"] = df_new["new_total"].cumsum()
df_new["source"] = "fhi:git"
df_new = df_new.sort_values(by=["date"], ascending=True)
df = load_datafile("tested_lab", parse_dates=["date"])
if not df_new.equals(df):
print(now, "tested_lab: New update")
sourcefile = load_sources()
sourcefile["tested_lab.csv"]["last_updated"] = now
sourcefile["tested_lab.csv"]["pending_update"] = 1
write_sources(sourcefile)
write_datafile("tested_lab", df_new)
# Generate graph
graphs.tested_lab()
def update():
now = get_timestr()
cols = {
"År/Year": "year",
"Uke/week": "week",
"Tilfeller/cases": "new_confirmed",
}
data = get_data()
if data:
df_new = pd.read_excel(data.content, usecols=cols)
df_new = df_new.rename(columns=cols)
df_new = df_new[df_new["year"].isin([2021, 2022])]
df_new = df_new[["year", "week",
"new_confirmed"]].fillna(0).astype(int)
df_new = df_new.sort_values(by="week").reset_index(drop=True)
df_new["total_confirmed"] = df_new["new_confirmed"].cumsum()
df_new["source"] = "fhi:web"
df = load_datafile("omicron")
if not df_new.equals(df):
print(now, "omicron: New update")
sourcefile = load_sources()
sourcefile["omicron.csv"]["last_updated"] = now
sourcefile["omicron.csv"]["pending_update"] = 1
write_sources(sourcefile)
write_datafile("omicron", df_new)
def receive_arg():
"""Process all hyper-parameters and experiment settings.
Record in opts_dict."""
parser = argparse.ArgumentParser()
parser.add_argument(
'--opt_path', type=str, default='option.yml',
help='Path to option YAML file.'
)
args = parser.parse_args()
with open(args.opt_path, 'r') as fp:
opts_dict = yaml.load(fp, Loader=yaml.FullLoader)
opts_dict['opt_path'] = args.opt_path
if opts_dict['train']['exp_name'] == None:
opts_dict['train']['exp_name'] = utils.get_timestr()
opts_dict['train']['log_path'] = op.join(
"exp", opts_dict['train']['exp_name'], "log_test.log"
)
opts_dict['train']['checkpoint_save_path_pre'] = op.join(
"exp", opts_dict['train']['exp_name'], "ckp_"
)
opts_dict['test']['restore_iter'] = int(
opts_dict['test']['restore_iter']
)
opts_dict['test']['checkpoint_save_path'] = (
f"{opts_dict['train']['checkpoint_save_path_pre']}"
f"{opts_dict['test']['restore_iter']}"
'.pt'
)
return opts_dict
def update():
now = get_timestr()
datafile = get_fhi_datafile("data_covid19_msis_by_time_location")
df_new = pd.read_csv(datafile,
usecols=["date", "n", "location_name"],
parse_dates=["date"])
df_new = df_new.loc[(df_new["location_name"] == "Norge")]
df_new = df_new.filter(items=["date", "n"])
df_new = df_new[df_new.date >= "2020-02-21"]
df_new = df_new.rename(columns={"n": "new"})
df_new["total"] = df_new["new"].cumsum()
df_new["source"] = "fhi:git"
df_new = df_new.reset_index(drop=True)
df = load_datafile("confirmed", parse_dates=["date"])
df = df[df.date >= "2020-02-21"]
df_filter = df.loc[df["source"] == "fhi:git"]
df_filter = df_filter.reset_index(drop=True)
if not df_new.equals(df_filter):
print(now, "fhi_git.confirmed: New update")
df_new = df_new.merge(df, how="outer")
df_new = df_new.drop_duplicates(subset=["date"], keep="first")
second_last = df_new.iloc[-2:]
second_last_total = second_last.total.values[0]
last_total = second_last.total.values[1]
last_new = second_last.new.values[1]
if second_last_total > last_total:
newToday = last_new + (second_last_total - last_total)
df_new.iloc[-1:]["total"] = second_last_total
df_new.iloc[-1:]["new"] = newToday
sourcefile = load_sources()
sourcefile["confirmed.csv"]["last_updated"] = now
sourcefile["confirmed.csv"]["pending_update"] = 1
write_sources(sourcefile)
write_datafile("confirmed", df_new)
# Generate graph
graphs.confirmed()
def update():
now = get_timestr()
today = date.today()
yesterday = today - timedelta(days=1)
url = "https://statistikk.fhi.no/api/msis/antallKoronaTotalt"
df = load_datafile("confirmed")
last_total = df["total"].max()
try:
confirmed_total = requests.get(url).json()
except Exception:
confirmed_total = 0
error = sys.exc_info()[1]
print(now, "- ERROR:", str(error))
if confirmed_total > last_total:
print(now, "msis_api.confirmed: New update")
confirmed_diff = confirmed_total - last_total
if datetime.now().hour in range(0, 2):
n_yesterday = df.new.loc[df["date"] == str(yesterday)].values[0]
diff_yesterday = n_yesterday + confirmed_diff
df.loc[df["date"] == str(yesterday), "new"] = diff_yesterday
df.loc[df["date"] == str(yesterday), "total"] = confirmed_total
df.loc[df["date"] == str(today), "total"] = confirmed_total
df.loc[df["date"] == str(yesterday), "source"] = "msis:api"
df.loc[df["date"] == str(today), "source"] = "msis:api"
else:
n_today = df.new.loc[df["date"] == str(today)].values[0]
diff_today = n_today + confirmed_diff
df.loc[df["date"] == str(today), "new"] = diff_today
df.loc[df["date"] == str(today), "total"] = confirmed_total
df.loc[df["date"] == str(today), "source"] = "msis:api"
sourcefile = load_sources()
sourcefile["confirmed.csv"]["last_updated"] = now
sourcefile["confirmed.csv"]["pending_update"] = 1
write_sources(sourcefile)
write_datafile("confirmed", df)
def update():
now = get_timestr()
url = "https://api.helsedirektoratet.no/ProduktCovid19/Covid19Statistikk/"
headers = {
"Ocp-Apim-Subscription-Key": os.getenv("HELSEDIR_API_KEY"),
}
res = requests.get(url + "helseregion", headers=headers).json()
df_new = pd.DataFrame()
for r in res:
df_new = df_new.append(r["registreringer"])
df_new = df_new.rename(
columns={
"dato": "date",
"antInnlagte": "admissions",
"antCovidIntensiv": "icu",
"antRespirator": "respiratory",
})
df_new = df_new[["date", "admissions", "icu", "respiratory"]]
df_new["date"] = pd.to_datetime(df_new["date"], format="%Y-%m-%d")
df_new = df_new.groupby("date").sum()
df_new = df_new.reset_index().sort_values(["date"], ascending=True)
df_new["source"] = "helsedir:api"
df = load_datafile("hospitalized", parse_dates=["date"])
if not df_new.equals(df):
print(now, "hospitalized: New update")
sourcefile = load_sources()
sourcefile["hospitalized.csv"]["last_updated"] = now
sourcefile["hospitalized.csv"]["pending_update"] = 1
write_sources(sourcefile)
write_datafile("hospitalized", df_new)
# Generate graph
graphs.hospitalized()
def receive_arg():
"""Process all hyper-parameters and experiment settings.
Record in opts_dict."""
parser = argparse.ArgumentParser()
parser.add_argument(
'--opt_path', type=str, default='option_R3_mfqev2_4G.yml',
help='Path to option YAML file.'
)
parser.add_argument(
'--local_rank', type=int, default=0,
help='Distributed launcher requires.'
)
args = parser.parse_args()
with open(args.opt_path, 'r') as fp:
opts_dict = yaml.load(fp, Loader=yaml.FullLoader)
opts_dict['opt_path'] = args.opt_path
opts_dict['train']['rank'] = args.local_rank
if opts_dict['train']['exp_name'] == None:
opts_dict['train']['exp_name'] = utils.get_timestr()
opts_dict['train']['log_path'] = op.join(
"exp", opts_dict['train']['exp_name'], "log.log"
)
opts_dict['train']['checkpoint_save_path_pre'] = op.join(
"exp", opts_dict['train']['exp_name'], "ckp_"
)
opts_dict['train']['num_gpu'] = torch.cuda.device_count()
if opts_dict['train']['num_gpu'] > 1:
opts_dict['train']['is_dist'] = True
else:
opts_dict['train']['is_dist'] = False
opts_dict['test']['restore_iter'] = int(
opts_dict['test']['restore_iter']
)
return opts_dict
def update():
now = get_timestr()
# load current data
df = load_datafile("dead")
# get fhi datafile
datafile = get_fhi_datafile("data_covid19_demographics")
df_new = pd.read_csv(datafile)
date_of_publishing = df_new.date_of_publishing.max()
if date_of_publishing not in df.date.values:
print(now, "dead: New update")
last_data = df["total"].max()
fhi_dead = df_new["n"].sum()
dead_diff = fhi_dead - last_data
df = df.append(
{
"date": date_of_publishing,
"new": dead_diff,
"total": fhi_dead,
"source": "fhi:git",
},
ignore_index=True,
)
sourcefile = load_sources()
sourcefile["dead.csv"]["last_updated"] = now
sourcefile["dead.csv"]["pending_update"] = 1
write_sources(sourcefile)
write_datafile("dead", df)
# Generate graph
graphs.dead()
def update():
now = get_timestr()
# get from fhi api
url = "https://www.fhi.no/api/chartdata/api/91672"
res = requests.get(url).json()
tests = res["figures"][4]
fhi_tests = tests["number"]
fhi_date = str(datetime.strptime(tests["updated"], "%d/%m/%Y").date())
# get current data
df = load_datafile("tested")
# update new data
if fhi_date not in df.date.values:
print(now, "tested: New update")
last_data = df["total"].max()
tested_diff = fhi_tests - last_data
df = df.append(
{
"date": fhi_date,
"new": tested_diff,
"total": fhi_tests,
"source": "fhi:web",
},
ignore_index=True,
)
sourcefile = load_sources()
sourcefile["tested.csv"]["last_updated"] = now
sourcefile["tested.csv"]["pending_update"] = 1
write_sources(sourcefile)
write_datafile("tested", df)
def update():
now = get_timestr()
url = "https://www.fhi.no/om/smittestopp/nokkeltall-fra-smittestopp/"
browser = get_browser()
try:
browser.get(url)
time.sleep(1)
new_downloads = browser.execute_script(
"return Highcharts.charts[0].series[0].options.data")
total_downloads = browser.execute_script(
"return Highcharts.charts[0].series[1].options.data")
new_reported = browser.execute_script(
"return Highcharts.charts[1].series[0].options.data")
total_reported = browser.execute_script(
"return Highcharts.charts[1].series[1].options.data")
browser.close()
browser.quit()
except Exception:
error = sys.exc_info()[1]
print("- ERROR:", str(error))
new_lst = []
for n in new_downloads:
date_parsed = datetime.fromtimestamp(n[0] / 1000)
datestr = date_parsed.strftime("%Y-%m-%d")
d = {"date": datestr, "new_downloads": n[1]}
new_lst.append(d)
df_new = pd.DataFrame(new_lst)
for n in total_downloads:
date_parsed = datetime.fromtimestamp(n[0] / 1000)
datestr = date_parsed.strftime("%Y-%m-%d")
df_new.loc[df_new["date"] == datestr, "total_downloads"] = n[1]
for n in new_reported:
date_parsed = datetime.fromtimestamp(n[0] / 1000)
datestr = date_parsed.strftime("%Y-%m-%d")
df_new.loc[df_new["date"] == datestr, "new_reported"] = n[1]
for n in total_reported:
date_parsed = datetime.fromtimestamp(n[0] / 1000)
datestr = date_parsed.strftime("%Y-%m-%d")
df_new.loc[df_new["date"] == datestr, "total_reported"] = n[1]
df_new = df_new.fillna(0)
intcolumns = [
"new_downloads",
"total_downloads",
"new_reported",
"total_reported",
]
df_new[intcolumns] = df_new[intcolumns].astype(int)
df_new["source"] = "fhi:web"
df_new["date"] = pd.to_datetime(df_new["date"])
columns = [
"date",
"new_downloads",
"total_downloads",
"new_reported",
"total_reported",
]
# Compare dfs
df = load_datafile("smittestopp", parse_dates=["date"])
df_new = df_new.sort_values(by=["date"])
merged = pd.merge(df_new, df, how="outer", indicator=True)
new_data = merged.loc[merged._merge == "left_only", columns]
if not new_data.empty:
print(now, "smittestopp: New update")
sourcefile = load_sources()
sourcefile["smittestopp.csv"]["last_updated"] = now
sourcefile["smittestopp.csv"]["pending_update"] = 1
write_sources(sourcefile)
write_datafile("smittestopp", df_new)
# Generate graph
graphs.smittestopp()
def update():
now = get_timestr()
# load current data
df = load_datafile("vaccine_doses")
# get fhi datafile
datafile = get_fhi_datafile("data_covid19_sysvak_by_time_location")
df_new = pd.read_csv(
datafile,
usecols=[
"date",
"granularity_geo",
"location_name",
"n_dose_1",
"n_dose_2",
"cum_n_dose_1",
"cum_n_dose_2",
"cum_pr100_dose_1",
"cum_pr100_dose_2",
],
)
mapping = {
"n_dose_1": "new_dose_1",
"n_dose_2": "new_dose_2",
"cum_n_dose_1": "total_dose_1",
"cum_n_dose_2": "total_dose_2",
"cum_pr100_dose_1": "total_pr100_dose_1",
"cum_pr100_dose_2": "total_pr100_dose_2",
}
columns = [
"granularity_geo",
"location_name",
"date",
"new_dose_1",
"new_dose_2",
"total_dose_1",
"total_dose_2",
"total_pr100_dose_1",
"total_pr100_dose_2",
"new_doses",
"total_doses",
"source",
]
df_new = df_new.rename(columns=mapping)
df_new["new_doses"] = df_new["new_dose_1"] + df_new["new_dose_2"]
df_new["total_doses"] = df_new["total_dose_1"] + df_new["total_dose_2"]
df_new["source"] = "fhi:git"
df_new = df_new[columns]
for column in columns:
df_new[column] = df_new[column].astype(str).replace("nan", "")
df[column] = df[column].astype(str)
df_new = df_new.sort_values(
by=["granularity_geo", "location_name", "date"]).reset_index(drop=True)
df = df.sort_values(
by=["granularity_geo", "location_name", "date"]).reset_index(drop=True)
if not df_new.index.equals(df.index):
print(now, "vaccine_doses: New update")
sourcefile = load_sources()
sourcefile["vaccine_doses.csv"]["last_updated"] = now
sourcefile["vaccine_doses.csv"]["pending_update"] = 1
write_sources(sourcefile)
write_datafile("vaccine_doses", df_new)
# Generate graph
graphs.vaccine_doses()
def update():
now = get_timestr()
headers = {
"Accept":
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"User-Agent":
"Mozilla/5.0 (Macintosh; Intel Mac OS X 11.2; rv:86.0) Gecko/20100101 Firefox/86.0",
"X-Requested-With": "XMLHttpRequest",
}
columns = [
"tr_type",
"route",
"company",
"tr_from",
"tr_to",
"departure",
"arrival",
"source",
]
df_new = pd.DataFrame(columns=columns)
base_url = "https://www.fhi.no"
url = f"{base_url}/sv/smittsomme-sykdommer/corona/koronavirus-og-covid-19-pa-offentlig-kommunikasjon/"
html = requests.get(url).text
soup = BeautifulSoup(html, "lxml")
for i in soup.find_all("div", class_="fhi-dynamic-table__download"):
a = i.find("a")
href = a.get("href")
filename = os.path.basename(href)
tr_types = {
"båt": "Skip",
"bat": "Skip",
"buss": "Buss",
"fly": "Fly",
"tog": "Tog",
}
res = requests.get(f"{base_url}{href}",
allow_redirects=True,
headers=headers)
df_tmp = pd.read_excel(res.content)
df_tmp = df_tmp.dropna()
for i, row in df_tmp.iterrows():
route = row[0]
company = row[1]
from_to = row[2].replace("\xa0", "")
departure = fmt_date(row[3])
arrival = fmt_date(row[4])
from_to_split = from_to.split("-")
_from = from_to_split[0].strip()
try:
_to = from_to_split[1].strip()
except IndexError:
_to = ""
tr_type = ""
try:
tr_type = filename.split("-")[1].split(".")[0]
except Exception:
raise Exception
values = {
"tr_type": tr_types[tr_type],
"route": route,
"company": company,
"tr_from": _from,
"tr_to": _to,
"departure": departure,
"arrival": arrival,
"source": "fhi:web",
}
df_new = df_new.append(pd.DataFrame(values, index=[0]))
df_new.loc[df_new.departure == "2021-12-27 16:05:00",
["departure"]] = "2020-12-27 16:05:00"
df_new["departure"] = pd.to_datetime(df_new["departure"])
df_new["arrival"] = pd.to_datetime(df_new["arrival"])
for column in columns:
df_new[column] = df_new[column].astype(str)
df_new = df_new.sort_values(by=["departure", "arrival", "route"],
ascending=False)
columns = [
"tr_type",
"route",
"company",
"tr_from",
"tr_to",
"departure",
"arrival",
"source",
]
df = load_datafile("transport")
for column in columns:
df_new[column] = df_new[column].astype(str).replace("nan", "")
df[column] = df[column].astype(str).replace("nan", "")
merged = pd.merge(df_new, df, how="outer", indicator=True)
new_data = merged.loc[merged._merge == "left_only", columns]
if not new_data.empty:
print(now, "transport: New update")
df = pd.concat([df, new_data])
df = df.sort_values(by=["departure", "arrival", "route"],
ascending=False)
sourcefile = load_sources()
sourcefile["transport.csv"]["last_updated"] = now
sourcefile["transport.csv"]["pending_update"] = 1
write_sources(sourcefile)
write_datafile("transport", df)