with open(parse.parsedfile, 'w') as outf: cw = csv.writer(outf) cw.writerow(['Area', 'Region', 'Date', 'Confirmed']) group = 'Métropole' for tr in tab.find('tbody').find_all('tr'): tds = tr.find_all('td') area = tds[0].get_text() counttxt = tds[1].get_text() if '**' in counttxt: continue count = cleannum(tds[1].get_text()) if area == 'Total Métropole': group = 'Outre Mer' continue if area == 'Total Outre Mer': area = '???' continue cw.writerow([area, group, parse.parsedtime, count]) parse.diff() if args.only_changed: if not parse.parseddiff.changed: print("parsed content \"%s\" unchanged" % parse.label) #return parse.deploy_day() fetchhelper.git_commit([parse], args)
jdat = json.loads(updatelk.rawdata) parselk = fetchhelper.ParseData(updatelk, 'lk') parselk.parsedtime = None with open(parselk.parsedfile, 'w') as outf: cout = csv.writer(outf) cout.writerow(['Area', 'Bundesland', 'AGS', 'Timestamp', 'Confirmed', 'Deaths']) for jfeat in sorted(jdat['features'], key=(lambda f: (f['attributes']['BL'], f['attributes']['GEN']))): ts_str = jfeat['attributes']['last_update'] if not ts_str: continue ts = datetime.datetime.strptime(jfeat['attributes']['last_update'], "%d.%m.%Y, %H:%M Uhr").astimezone(datatz) if parselk.parsedtime is None or ts > parselk.parsedtime: parselk.parsedtime = ts area = jfeat['attributes']['GEN'] if area in label_areas: area += ' (%s)' % bez_label[jfeat['attributes']['BEZ']] cout.writerow([ area, jfeat['attributes']['BL'], jfeat['attributes']['AGS_0'], ts.isoformat(), jfeat['attributes']['cases'], jfeat['attributes']['deaths'], ]) parselk.deploy_timestamp() fetchhelper.git_commit([parsebl, parselk], args)
cols.append(datatime.isoformat()) if tab_combined: cols += [ clean_num(tds[1].get_text()), clean_num(tds[n_deaths].get_text()) ] else: cols.append(clean_num(tds[1].get_text())) if deaths: cols.append(deaths.get(lk, 0)) cout.writerow(cols) # If the current day is later than the contenttime we assume the # content time is a mistake and we adjust it to the current day. # (This problem has happend before) #if parse.update.rawtime.date() > parse.parsedtime.date(): # if parse.parseddiff.changed and not parse.parseddiff.first: # print("Adjust date", parse.parsedtime, "->", parse.update.rawtime) # parse.parsedtime = parse.update.rawtime parse.deploy_timestamp() rparse = fetchhelper.ParseData(update, 'regierungsbezirk') parse_table(rparse, html, 'regierungsbezirk') lparse = fetchhelper.ParseData(update, 'landkreis') parse_table(lparse, html, 'landkreis') fetchhelper.git_commit([rparse, lparse], args)
header = jdat['data'][0][0] i_kom = header.index("Ort") i_rec = 1 assert (header[i_rec].startswith("Stand ")) parse = fetchhelper.ParseData(update, 'recovered') parse.parsedtime = datetime.datetime.fromtimestamp(jdat['refreshed'] / 1000, tz=datatz) with open(parse.parsedfile, 'w') as f: cw = csv.writer(f) cw.writerow(['Kommune', 'Timestamp', 'Recovered']) for jrow in jdat['data'][0][1:]: if jrow[i_kom] in ('Zuordnung fehlt', 'Gesamt', ''): continue if jrow[i_kom] == 'Pfaffenhofen a.d.Glonn': jrow[i_kom] = 'Pfaffenhofen a.d. Glonn' cw.writerow([jrow[i_kom], parse.parsedtime.isoformat(), jrow[i_rec]]) parse.deploy_timestamp() if fetchhelper.csv_equal(parse.deployfile, parse.deployfile_previous(), skip=['Timestamp']): os.unlink(parse.deployfile) else: parses.append(parse) if parses: fetchhelper.git_commit(parses, args)