def retrieveLatestCsvDate(args): metaDataUrl = 'https://www.arcgis.com/sharing/rest/content/items/f10774f1c63e40168479a1feb6c7ca74?f=json' response = requestURL(metaDataUrl) if hasattr(response,'reason') and response.reason != "OK": print("response.reason", response.reason) return None else: if hasattr(response, 'code'): if response.code == 200: print("response.code", response.code) metaData = json.loads(response.read()) pmu.pretty(metaData) lastModDateStr = metaData["modified"] lastModeDate = cd.datetimeFromStampStr(lastModDateStr) print(lastModDateStr, lastModeDate) return lastModeDate return None
def unify(table): dss = table[0, "Datenstand"] ds = cd.datetimeFromDatenstandAny(dss) dsdy = cd.dayFromDate(ds) hasRefdatum = "Refdatum" in table.names hasErkrankungsbeginn = "IstErkrankungsbeginn" in table.names #t = table.copy() t = table if "Altersgruppe2" in table.names: t = t[:, dt.f[:].remove(dt.f["Altersgruppe2"])] if not "DatenstandISO" in table.names: isodate = cd.dateStrYMDFromDay(dsdy) t = t[:, dt.f[:].extend({"DatenstandISO": isodate})] if not hasRefdatum: t = t[:, dt.f[:].extend({ "Refdatum": str(cd.day0d), "RefdatumISO": dt.f.MeldedatumISO })] if not hasErkrankungsbeginn: t = t[:, dt.f[:].extend({"IstErkrankungsbeginn": 0})] if "NeuGenesen" not in table.names: t = t[:, dt.f[:].extend({"NeuGenesen": -9, "AnzahlGenesen": 0})] t = t[:, dt.f[:].extend({ "FallGruppe": "", "MeldeTag": nan, "RefTag": nan, "DatenstandTag": dsdy })] #t = t[:, dt.f[:].extend({"Bevoelkerung":0, "FaellePro100k":0.0, "TodesfaellePro100k":0.0, "isStadt":False})] #t = t[:, dt.f[:].extend({"Flaeche":0.0, "FaelleProKm2":0.0, "TodesfaelleProKm2":0.0, "Dichte":0.0})] #print("unified fields", t.names) #Bevoelkerung = loadLandkreisBeveolkerung() #Flaeche = loadLandkreisFlaeche() #Census = loadCensus() #pmu.printMemoryUsage("unify pre realize ") #t.materialize(to_memory=True) pmu.printMemoryUsage("unify pre dict") d = t.to_dict() pmu.printMemoryUsage("unify post dict") print("> iterating through {} rows".format(t.nrows)) start = time.perf_counter() for r in range(t.nrows): mds = d["Meldedatum"][r] if pmu.is_int(mds): md = cd.datetimeFromStampStr(mds) else: md = datetimeFromDateStr3(mds) mdy = cd.dayFromDate(md) d["MeldeTag"][r] = mdy if not hasRefdatum: d["Refdatum"][r] = str(md) d["RefTag"][r] = mdy fg = str(d["IdLandkreis"] [r]) + d["Altersgruppe"][r] + d["Geschlecht"][r] + str( int(d["MeldeTag"][r])) if int(d["IstErkrankungsbeginn"][r]) == 1: rds = d["Refdatum"][r] if pmu.is_int(rds): rd = cd.datetimeFromStampStr(rds) else: rd = datetimeFromDateStr3(rds) rdy = cd.dayFromDate(rd) d["RefTag"][r] = rdy fg = fg + ":" + str(rdy) d["FallGruppe"][r] = fg checkLandkreisData(d, r, Census, Flaeche) finish = time.perf_counter() print("< iterating through {} rows done, {:.1f} rows/sec".format( t.nrows, t.nrows / (finish - start))) pmu.printMemoryUsage("end of unify, pre frame") t = dt.Frame(d) pmu.printMemoryUsage("end of unify, post frame") return t
def unify(table, makeFallGruppe=False): dss = table[0, "Datenstand"] ds = cd.datetimeFromDatenstandAny(dss) if 'FID' in table.names: table.names = {"FID": "ObjectId"} dsdy = cd.dayFromDate(ds) dsisodate = cd.dateStrYMDFromDay(dsdy) hasRefdatum = "Refdatum" in table.names hasErkrankungsbeginn = "IstErkrankungsbeginn" in table.names #t = table.copy() t = table if "Altersgruppe2" in table.names: t = t[:, dt.f[:].remove(dt.f["Altersgruppe2"])] if not "DatenstandISO" in table.names: t = t[:, dt.f[:].extend({"DatenstandISO": dsisodate})] if not hasRefdatum: t = t[:, dt.f[:].extend({"Refdatum": 0})] hasRefdatumISO = "RefdatumISO" in table.names if not hasRefdatumISO: #print("t1",t.names) t = t[:, dt.f[:].extend({"RefdatumISO": ""})] #print("t2",t.names) hasMeldedatumISO = "MeldedatumISO" in table.names if not hasMeldedatumISO: t = t[:, dt.f[:].extend({"MeldedatumISO": ""})] if not hasErkrankungsbeginn: t = t[:, dt.f[:].extend({"IstErkrankungsbeginn": 0})] if "NeuGenesen" not in table.names: t = t[:, dt.f[:].extend({"NeuGenesen": -9, "AnzahlGenesen": 0})] if makeFallGruppe: t = t[:, dt.f[:].extend({ "FallGruppe": "", "MeldeTag": nan, "RefTag": nan, "DatenstandTag": dsdy })] t = t[:, dt.f[:].extend({ "MeldeTag": nan, "RefTag": nan, "DatenstandTag": dsdy })] pmu.printMemoryUsage("unify pre dict") d = t.to_dict() pmu.printMemoryUsage("unify post dict") print("> iterating through {} rows".format(t.nrows)) start = time.perf_counter() for r in range(t.nrows): mds = d["Meldedatum"][r] if pmu.is_int(mds): md = cd.datetimeFromStampStr(mds) else: md = datetimeFromDateStr3(mds) d["Meldedatum"][r] = ticksFromDateTime(md) mdy = cd.dayFromDate(md) d["MeldeTag"][r] = mdy if not hasRefdatum: d["Refdatum"][r] = ticksFromDateTime(md) d["RefTag"][r] = mdy if not hasMeldedatumISO: d["MeldedatumISO"][r] = cd.dateStrYMDFromDay(mdy) if makeFallGruppe: fg = str(d["IdLandkreis"] [r]) + d["Altersgruppe"][r] + d["Geschlecht"][r] + str( int(d["MeldeTag"][r])) #if int(d["IstErkrankungsbeginn"][r]) == 1: rds = d["Refdatum"][r] if pmu.is_int(rds): rd = cd.datetimeFromStampStr(rds) else: rd = datetimeFromDateStr3(rds) d["Refdatum"][r] = ticksFromDateTime(rd) rdy = cd.dayFromDate(rd) d["RefTag"][r] = rdy if not hasRefdatumISO: d["RefdatumISO"][r] = cd.dateStrYMDFromDay(rdy) if makeFallGruppe: fg = fg + ":" + str(rdy) if makeFallGruppe: d["FallGruppe"][r] = fg checkLandkreisData(d, r, Census, Flaeche) finish = time.perf_counter() print("< iterating through {} rows done, {:.1f} rows/sec".format( t.nrows, t.nrows / (finish - start))) pmu.printMemoryUsage("end of unify, pre frame") t = dt.Frame(d) pmu.printMemoryUsage("end of unify, post frame") return t