def retrieveLatestCsvDate(args):
metaDataUrl = 'https://www.arcgis.com/sharing/rest/content/items/f10774f1c63e40168479a1feb6c7ca74?f=json'
response = requestURL(metaDataUrl)
if hasattr(response,'reason') and response.reason != "OK":
print("response.reason", response.reason)
return None
else:
if hasattr(response, 'code'):
if response.code == 200:
print("response.code", response.code)
metaData = json.loads(response.read())
pmu.pretty(metaData)
lastModDateStr = metaData["modified"]
lastModeDate = cd.datetimeFromStampStr(lastModDateStr)
print(lastModDateStr, lastModeDate)
return lastModeDate
return None
def unify(table):
dss = table[0, "Datenstand"]
ds = cd.datetimeFromDatenstandAny(dss)
dsdy = cd.dayFromDate(ds)
hasRefdatum = "Refdatum" in table.names
hasErkrankungsbeginn = "IstErkrankungsbeginn" in table.names
#t = table.copy()
t = table
if "Altersgruppe2" in table.names:
t = t[:, dt.f[:].remove(dt.f["Altersgruppe2"])]
if not "DatenstandISO" in table.names:
isodate = cd.dateStrYMDFromDay(dsdy)
t = t[:, dt.f[:].extend({"DatenstandISO": isodate})]
if not hasRefdatum:
t = t[:, dt.f[:].extend({
"Refdatum": str(cd.day0d),
"RefdatumISO": dt.f.MeldedatumISO
})]
if not hasErkrankungsbeginn:
t = t[:, dt.f[:].extend({"IstErkrankungsbeginn": 0})]
if "NeuGenesen" not in table.names:
t = t[:, dt.f[:].extend({"NeuGenesen": -9, "AnzahlGenesen": 0})]
t = t[:, dt.f[:].extend({
"FallGruppe": "",
"MeldeTag": nan,
"RefTag": nan,
"DatenstandTag": dsdy
})]
#t = t[:, dt.f[:].extend({"Bevoelkerung":0, "FaellePro100k":0.0, "TodesfaellePro100k":0.0, "isStadt":False})]
#t = t[:, dt.f[:].extend({"Flaeche":0.0, "FaelleProKm2":0.0, "TodesfaelleProKm2":0.0, "Dichte":0.0})]
#print("unified fields", t.names)
#Bevoelkerung = loadLandkreisBeveolkerung()
#Flaeche = loadLandkreisFlaeche()
#Census = loadCensus()
#pmu.printMemoryUsage("unify pre realize ")
#t.materialize(to_memory=True)
pmu.printMemoryUsage("unify pre dict")
d = t.to_dict()
pmu.printMemoryUsage("unify post dict")
print("> iterating through {} rows".format(t.nrows))
start = time.perf_counter()
for r in range(t.nrows):
mds = d["Meldedatum"][r]
if pmu.is_int(mds):
md = cd.datetimeFromStampStr(mds)
else:
md = datetimeFromDateStr3(mds)
mdy = cd.dayFromDate(md)
d["MeldeTag"][r] = mdy
if not hasRefdatum:
d["Refdatum"][r] = str(md)
d["RefTag"][r] = mdy
fg = str(d["IdLandkreis"]
[r]) + d["Altersgruppe"][r] + d["Geschlecht"][r] + str(
int(d["MeldeTag"][r]))
if int(d["IstErkrankungsbeginn"][r]) == 1:
rds = d["Refdatum"][r]
if pmu.is_int(rds):
rd = cd.datetimeFromStampStr(rds)
else:
rd = datetimeFromDateStr3(rds)
rdy = cd.dayFromDate(rd)
d["RefTag"][r] = rdy
fg = fg + ":" + str(rdy)
d["FallGruppe"][r] = fg
checkLandkreisData(d, r, Census, Flaeche)
finish = time.perf_counter()
print("< iterating through {} rows done, {:.1f} rows/sec".format(
t.nrows, t.nrows / (finish - start)))
pmu.printMemoryUsage("end of unify, pre frame")
t = dt.Frame(d)
pmu.printMemoryUsage("end of unify, post frame")
return t
def unify(table, makeFallGruppe=False):
dss = table[0, "Datenstand"]
ds = cd.datetimeFromDatenstandAny(dss)
if 'FID' in table.names:
table.names = {"FID": "ObjectId"}
dsdy = cd.dayFromDate(ds)
dsisodate = cd.dateStrYMDFromDay(dsdy)
hasRefdatum = "Refdatum" in table.names
hasErkrankungsbeginn = "IstErkrankungsbeginn" in table.names
#t = table.copy()
t = table
if "Altersgruppe2" in table.names:
t = t[:, dt.f[:].remove(dt.f["Altersgruppe2"])]
if not "DatenstandISO" in table.names:
t = t[:, dt.f[:].extend({"DatenstandISO": dsisodate})]
if not hasRefdatum:
t = t[:, dt.f[:].extend({"Refdatum": 0})]
hasRefdatumISO = "RefdatumISO" in table.names
if not hasRefdatumISO:
#print("t1",t.names)
t = t[:, dt.f[:].extend({"RefdatumISO": ""})]
#print("t2",t.names)
hasMeldedatumISO = "MeldedatumISO" in table.names
if not hasMeldedatumISO:
t = t[:, dt.f[:].extend({"MeldedatumISO": ""})]
if not hasErkrankungsbeginn:
t = t[:, dt.f[:].extend({"IstErkrankungsbeginn": 0})]
if "NeuGenesen" not in table.names:
t = t[:, dt.f[:].extend({"NeuGenesen": -9, "AnzahlGenesen": 0})]
if makeFallGruppe:
t = t[:, dt.f[:].extend({
"FallGruppe": "",
"MeldeTag": nan,
"RefTag": nan,
"DatenstandTag": dsdy
})]
t = t[:, dt.f[:].extend({
"MeldeTag": nan,
"RefTag": nan,
"DatenstandTag": dsdy
})]
pmu.printMemoryUsage("unify pre dict")
d = t.to_dict()
pmu.printMemoryUsage("unify post dict")
print("> iterating through {} rows".format(t.nrows))
start = time.perf_counter()
for r in range(t.nrows):
mds = d["Meldedatum"][r]
if pmu.is_int(mds):
md = cd.datetimeFromStampStr(mds)
else:
md = datetimeFromDateStr3(mds)
d["Meldedatum"][r] = ticksFromDateTime(md)
mdy = cd.dayFromDate(md)
d["MeldeTag"][r] = mdy
if not hasRefdatum:
d["Refdatum"][r] = ticksFromDateTime(md)
d["RefTag"][r] = mdy
if not hasMeldedatumISO:
d["MeldedatumISO"][r] = cd.dateStrYMDFromDay(mdy)
if makeFallGruppe:
fg = str(d["IdLandkreis"]
[r]) + d["Altersgruppe"][r] + d["Geschlecht"][r] + str(
int(d["MeldeTag"][r]))
#if int(d["IstErkrankungsbeginn"][r]) == 1:
rds = d["Refdatum"][r]
if pmu.is_int(rds):
rd = cd.datetimeFromStampStr(rds)
else:
rd = datetimeFromDateStr3(rds)
d["Refdatum"][r] = ticksFromDateTime(rd)
rdy = cd.dayFromDate(rd)
d["RefTag"][r] = rdy
if not hasRefdatumISO:
d["RefdatumISO"][r] = cd.dateStrYMDFromDay(rdy)
if makeFallGruppe:
fg = fg + ":" + str(rdy)
if makeFallGruppe:
d["FallGruppe"][r] = fg
checkLandkreisData(d, r, Census, Flaeche)
finish = time.perf_counter()
print("< iterating through {} rows done, {:.1f} rows/sec".format(
t.nrows, t.nrows / (finish - start)))
pmu.printMemoryUsage("end of unify, pre frame")
t = dt.Frame(d)
pmu.printMemoryUsage("end of unify, post frame")
return t