with open(parse.parsedfile, 'w') as outf:
cw = csv.writer(outf)
cw.writerow(['Area', 'Region', 'Date', 'Confirmed'])
group = 'Métropole'
for tr in tab.find('tbody').find_all('tr'):
tds = tr.find_all('td')
area = tds[0].get_text()
counttxt = tds[1].get_text()
if '**' in counttxt:
continue
count = cleannum(tds[1].get_text())
if area == 'Total Métropole':
group = 'Outre Mer'
continue
if area == 'Total Outre Mer':
area = '???'
continue
cw.writerow([area, group, parse.parsedtime, count])
parse.diff()
if args.only_changed:
if not parse.parseddiff.changed:
print("parsed content \"%s\" unchanged" % parse.label)
#return
parse.deploy_day()
fetchhelper.git_commit([parse], args)
jdat = json.loads(updatelk.rawdata)
parselk = fetchhelper.ParseData(updatelk, 'lk')
parselk.parsedtime = None
with open(parselk.parsedfile, 'w') as outf:
cout = csv.writer(outf)
cout.writerow(['Area', 'Bundesland', 'AGS', 'Timestamp', 'Confirmed', 'Deaths'])
for jfeat in sorted(jdat['features'], key=(lambda f: (f['attributes']['BL'], f['attributes']['GEN']))):
ts_str = jfeat['attributes']['last_update']
if not ts_str:
continue
ts = datetime.datetime.strptime(jfeat['attributes']['last_update'], "%d.%m.%Y, %H:%M Uhr").astimezone(datatz)
if parselk.parsedtime is None or ts > parselk.parsedtime:
parselk.parsedtime = ts
area = jfeat['attributes']['GEN']
if area in label_areas:
area += ' (%s)' % bez_label[jfeat['attributes']['BEZ']]
cout.writerow([
area,
jfeat['attributes']['BL'],
jfeat['attributes']['AGS_0'],
ts.isoformat(),
jfeat['attributes']['cases'],
jfeat['attributes']['deaths'],
])
parselk.deploy_timestamp()
fetchhelper.git_commit([parsebl, parselk], args)
cols.append(datatime.isoformat())
if tab_combined:
cols += [
clean_num(tds[1].get_text()),
clean_num(tds[n_deaths].get_text())
]
else:
cols.append(clean_num(tds[1].get_text()))
if deaths:
cols.append(deaths.get(lk, 0))
cout.writerow(cols)
# If the current day is later than the contenttime we assume the
# content time is a mistake and we adjust it to the current day.
# (This problem has happend before)
#if parse.update.rawtime.date() > parse.parsedtime.date():
# if parse.parseddiff.changed and not parse.parseddiff.first:
# print("Adjust date", parse.parsedtime, "->", parse.update.rawtime)
# parse.parsedtime = parse.update.rawtime
parse.deploy_timestamp()
rparse = fetchhelper.ParseData(update, 'regierungsbezirk')
parse_table(rparse, html, 'regierungsbezirk')
lparse = fetchhelper.ParseData(update, 'landkreis')
parse_table(lparse, html, 'landkreis')
fetchhelper.git_commit([rparse, lparse], args)
header = jdat['data'][0][0]
i_kom = header.index("Ort")
i_rec = 1
assert (header[i_rec].startswith("Stand "))
parse = fetchhelper.ParseData(update, 'recovered')
parse.parsedtime = datetime.datetime.fromtimestamp(jdat['refreshed'] / 1000,
tz=datatz)
with open(parse.parsedfile, 'w') as f:
cw = csv.writer(f)
cw.writerow(['Kommune', 'Timestamp', 'Recovered'])
for jrow in jdat['data'][0][1:]:
if jrow[i_kom] in ('Zuordnung fehlt', 'Gesamt', ''):
continue
if jrow[i_kom] == 'Pfaffenhofen a.d.Glonn':
jrow[i_kom] = 'Pfaffenhofen a.d. Glonn'
cw.writerow([jrow[i_kom], parse.parsedtime.isoformat(), jrow[i_rec]])
parse.deploy_timestamp()
if fetchhelper.csv_equal(parse.deployfile,
parse.deployfile_previous(),
skip=['Timestamp']):
os.unlink(parse.deployfile)
else:
parses.append(parse)
if parses:
fetchhelper.git_commit(parses, args)