def test_sd(): assert str(dt.sd(f.A)) == str(f.A.sd()) assert str(dt.sd(f[:])) == str(f[:].sd()) DT = dt.Frame({"C": [2, 5, 30, 20, 10], "D": [10, 8, 20, 20, 1]}) assert_equals(DT[:, f[:].sd()], DT[:, dt.sd(f[:])])
def test_rows_stdev(): from datatable import sd df0 = dt.Frame(range(10), names=["A"]) # stdev = 3.0276 df1 = df0(f.A > sd(f.A), engine="eager") assert df1.internal.check() assert df1.topython() == [[4, 5, 6, 7, 8, 9]]
def test_rows_stdev(): from datatable import sd df0 = dt.Frame(A=range(10)) # stdev = 3.0276 df1 = df0[f.A > sd(f.A), :] frame_integrity_check(df1) assert df1.to_list() == [[4, 5, 6, 7, 8, 9]]
fun=fun, run=2, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(flatten(chk.to_list())), chk_time_sec=chkt, on_disk=on_disk) print(ans.head(3), flush=True) print(ans.tail(3), flush=True) del ans question = 'median v3 sd v3 by id4 id5' # q6 gc.collect() t_start = timeit.default_timer() ans = x[:, {'median_v3': median(f.v3), 'sd_v3': sd(f.v3)}, by(f.id4, f.id5)] print(ans.shape, flush=True) t = timeit.default_timer() - t_start m = memory_usage() t_start = timeit.default_timer() chk = ans[:, [sum(f.median_v3), sum(f.sd_v3)]] chkt = timeit.default_timer() - t_start write_log(task=task, data=data_name, in_rows=x.shape[0], question=question, out_rows=ans.shape[0], out_cols=ans.shape[1], solution=solution, version=ver, git=git,
def analyzeDaily(fullTable, filter, postfix): #print("----- analyzeDaily:"+postfix) #dayTable = fullTable[(dt.f.DatenstandTag >= fromDay) & (dt.f.DatenstandTag < toDay) & (dt.f.IdLandkreis == forIdLandkreis),:] dayTable = fullTable[filter, :] cases_to_count = dayTable[(dt.f.NeuerFall == 0) | (dt.f.NeuerFall == 1), :] cases = cases_to_count[:, [dt.sum(dt.f.AnzahlFall)], dt.by(dt.f.DatenstandTag)] cases.names = ["DatenstandTag", "AnzahlFall" + postfix] cases.key = "DatenstandTag" print("cases rows = {}, cases_to_count = {}".format( cases.nrows, cases_to_count.nrows)) #print(cases) new_cases_to_count = dayTable[(dt.f.NeuerFall == -1) | (dt.f.NeuerFall == 1), :] new_cases = new_cases_to_count[:, [dt.sum(dt.f.AnzahlFall)], dt.by(dt.f.DatenstandTag)] new_cases.names = ["DatenstandTag", "AnzahlFallNeu" + postfix] new_cases.key = "DatenstandTag" print("new_cases rows = {}, new_cases_to_count = {}".format( new_cases.nrows, new_cases_to_count.nrows)) #new_cases_to_count.to_csv("new_cases_to_count.csv") new_cases_to_count_delay = new_cases_to_count[( dt.f.AnzahlFall > 0), :] # measure delay only for positive cases new_cases_to_count_delay.materialize() new_cases_delay = new_cases_to_count_delay[:, [ dt.min(dt.f.MeldeDelay), dt.max(dt.f.MeldeDelay), dt.mean(dt.f.MeldeDelay), dt.median(dt.f.MeldeDelay), dt.sd(dt.f.MeldeDelay), dt.sum(dt.f.AnzahlFall), dt.max(dt.f.DatenstandTag) ], dt.by(dt.f.DatenstandTag)] new_cases_delay.names = [ "DatenstandTag", "MeldeDauerFallNeu-Min" + postfix, "MeldeDauerFallNeu-Max" + postfix, "MeldeDauerFallNeu-Schnitt" + postfix, "MeldeDauerFallNeu-Median" + postfix, "MeldeDauerFallNeu-StdAbw" + postfix, "MeldeDauerFallNeu-Fallbasis" + postfix, "DatenstandTag-Max" + postfix ] new_cases_delay.key = "DatenstandTag" print("new_cases_delay rows = {}, new_cases_to_count_delay = {}".format( new_cases_delay.nrows, new_cases_to_count_delay.nrows)) #new_cases_delay = new_cases_to_count_delay[:, [dt.mean(dt.f.DatenstandTag-dt.f.MeldeTag)],dt.by(dt.f.DatenstandTag)] # delays = delayRecs[:, [dt.mean(dt.f.MeldeDelay), dt.median(dt.f.MeldeDelay), dt.sd(dt.f.MeldeDelay), dt.sum(dt.f.AnzahlFall)], dt.by(dt.f.Landkreis)] # new_cases_stddev = new_cases_to_count_delay[:, [dt.mean(dt.f.DatenstandTag - dt.f.MeldeTag)], # dt.by(dt.f.DatenstandTag)] # new_cases_delay.names = ["DatenstandTag", "AnzahlFallNeu-MeldeDauer" + postfix] # new_cases_delay.key = "DatenstandTag" # print("new_cases_delay rows = {}, new_cases_to_count_delay = {}".format(new_cases_delay.nrows, # new_cases_to_count_delay.nrows)) new_cases_to_count_strict = new_cases_to_count[( dt.f.DatenstandTag - dt.f.MeldeTag < 7) | (dt.f.AnzahlFall < 0), :] new_cases_strict = new_cases_to_count_strict[:, [dt.sum(dt.f.AnzahlFall)], dt.by(dt.f.DatenstandTag)] new_cases_strict.names = [ "DatenstandTag", "AnzahlFallNeu-Meldung-letze-7-Tage" + postfix ] new_cases_strict.key = "DatenstandTag" print("new_cases_strict rows = {}, new_cases_to_count_strict = {}".format( new_cases_strict.nrows, new_cases_to_count_strict.nrows)) #new_cases_to_count_strict.to_csv("new_cases_to_count_strict.csv") new_cases_to_count_strict_14 = new_cases_to_count[( dt.f.DatenstandTag - dt.f.MeldeTag < 14) | (dt.f.AnzahlFall < 0), :] new_cases_strict_14 = new_cases_to_count_strict_14[:, [ dt.sum(dt.f.AnzahlFall) ], dt.by(dt.f.DatenstandTag)] new_cases_strict_14.names = [ "DatenstandTag", "AnzahlFallNeu-Meldung-letze-14-Tage" + postfix ] new_cases_strict_14.key = "DatenstandTag" print("new_cases_strict_14 rows = {}, new_cases_to_count_strict_14 = {}". format(new_cases_strict_14.nrows, new_cases_to_count_strict_14.nrows)) dead_to_count = dayTable[(dt.f.NeuerTodesfall == 0) | (dt.f.NeuerTodesfall == 1), :] dead = dead_to_count[:, [dt.sum(dt.f.AnzahlTodesfall)], dt.by(dt.f.DatenstandTag)] dead.names = ["DatenstandTag", "AnzahlTodesfall" + postfix] dead.key = "DatenstandTag" #print("dead rows = {}".format(dead.nrows)) new_dead_to_count = dayTable[(dt.f.NeuerTodesfall == -1) | (dt.f.NeuerTodesfall == 1), :] new_dead = new_dead_to_count[:, [dt.sum(dt.f.AnzahlTodesfall)], dt.by(dt.f.DatenstandTag)] new_dead.names = ["DatenstandTag", "AnzahlTodesfallNeu" + postfix] new_dead.key = "DatenstandTag" #print("new_dead rows = {}".format(new_dead.nrows)) recovered_to_count = dayTable[(dt.f.NeuGenesen == 0) | (dt.f.NeuGenesen == 1), :] recovered = recovered_to_count[:, [dt.sum(dt.f.AnzahlGenesen)], dt.by(dt.f.DatenstandTag)] recovered.names = ["DatenstandTag", "AnzahlGenesen" + postfix] recovered.key = "DatenstandTag" #print("recovered rows = {}".format(recovered.nrows)) new_recovered_to_count = dayTable[(dt.f.NeuGenesen == -1) | (dt.f.NeuGenesen == 1), :] new_recovered = new_recovered_to_count[:, [dt.sum(dt.f.AnzahlGenesen)], dt.by(dt.f.DatenstandTag)] new_recovered.names = ["DatenstandTag", "AnzahlGenesenNeu" + postfix] new_recovered.key = "DatenstandTag" #print("new_recovered rows = {}".format(new_recovered.nrows)) byDayTable = cases[:,:,dt.join(new_cases)]\ [:,:,dt.join(dead)][:,:,dt.join(new_dead)][:,:,dt.join(recovered)][:,:,dt.join(new_recovered)]\ [:,:,dt.join(new_cases_strict)][:,:,dt.join(new_cases_strict_14)][:,:,dt.join(new_cases_delay)] byDayTable.key = "DatenstandTag" #print("byDayTable rows = {}".format(byDayTable.nrows)) print(byDayTable) return byDayTable
def analyzeDaily(fullTable, filter, prefix, postfix, byDateColName): print("analyzeDaily prefix='{}' postfix='{}' byDateColName='{}'".format(prefix, postfix, byDateColName)) #print("analyzeDaily filter='{}' '".format(filter)) byDate = dt.f[byDateColName] #print("----- analyzeDaily:"+postfix) #dayTable = fullTable[(dt.f.DatenstandTag >= fromDay) & (dt.f.DatenstandTag < toDay) & (dt.f.IdLandkreis == forIdLandkreis),:] dayTable = fullTable[filter,:] cases_to_count = dayTable[(dt.f.NeuerFall == 0) | (dt.f.NeuerFall == 1),:] cases = cases_to_count[:, [dt.sum(dt.f.AnzahlFall)],dt.by(byDate)] cases.names = [byDateColName, prefix+"AnzahlFall"+postfix] cases.key = byDateColName print("cases rows = {}, cases_to_count = {}".format(cases.nrows, cases_to_count.nrows)) #print(cases) byDayTable = cases if byDateColName == "DatenstandTag": new_cases_to_count = dayTable[(dt.f.NeuerFall == -1) | (dt.f.NeuerFall == 1),:] new_cases = new_cases_to_count[:, [dt.sum(dt.f.AnzahlFall)],dt.by(byDate)] new_cases.names = [byDateColName, prefix+"AnzahlFallNeu"+postfix] new_cases.key = byDateColName print("new_cases rows = {}, new_cases_to_count = {}".format(new_cases.nrows, new_cases_to_count.nrows)) #new_cases_to_count.to_csv("new_cases_to_count.csv") byDayTable = byDayTable[:,:,dt.join(new_cases)] else: # add days by MeldeTag byDayTable.names = {prefix+"AnzahlFall"+postfix: prefix+"AnzahlFallNeu"+postfix} byDayTable = addRunningSumColumn(byDayTable, prefix+"AnzahlFallNeu"+postfix, prefix+"AnzahlFall"+postfix) dead_to_count = dayTable[(dt.f.NeuerTodesfall == 0) | (dt.f.NeuerTodesfall == 1),:] dead = dead_to_count[:, [dt.sum(dt.f.AnzahlTodesfall)],dt.by(byDate)] dead.names = [byDateColName, prefix+"AnzahlTodesfall"+postfix] dead.key = byDateColName #print("dead rows = {}".format(dead.nrows)) byDayTable = byDayTable[:,:,dt.join(dead)] if byDateColName == "DatenstandTag": new_dead_to_count = dayTable[(dt.f.NeuerTodesfall == -1) | (dt.f.NeuerTodesfall == 1),:] new_dead = new_dead_to_count[:, [dt.sum(dt.f.AnzahlTodesfall)],dt.by(byDate)] new_dead.names = [byDateColName, prefix+"AnzahlTodesfallNeu"+postfix] new_dead.key = byDateColName #print("new_dead rows = {}".format(new_dead.nrows)) byDayTable = byDayTable[:,:,dt.join(new_dead)] else: # add days by MeldeTag byDayTable.names = {prefix+"AnzahlTodesfall"+postfix: prefix+"AnzahlTodesfallNeu"+postfix} byDayTable = addRunningSumColumn(byDayTable, prefix+"AnzahlTodesfallNeu"+postfix, prefix+"AnzahlTodesfall"+postfix) byDayTable.key = byDateColName if postfix == "" and prefix == "" and byDateColName == "DatenstandTag": new_cases_to_count_delay = new_cases_to_count[(dt.f.AnzahlFall > 0), :] # measure delay only for positive cases new_cases_to_count_delay.materialize() new_cases_delay = new_cases_to_count_delay[:, [dt.min(dt.f.MeldeDelay), dt.max(dt.f.MeldeDelay), dt.mean(dt.f.MeldeDelay), dt.median(dt.f.MeldeDelay), dt.sd(dt.f.MeldeDelay), dt.sum(dt.f.AnzahlFall), dt.max(dt.f.DatenstandTag)], dt.by(byDate)] new_cases_delay.names = ["DatenstandTag", "PublikationsdauerFallNeu_Min" + postfix, "PublikationsdauerFallNeu_Max" + postfix, "PublikationsdauerFallNeu_Schnitt" + postfix, "PublikationsdauerFallNeu_Median" + postfix, "PublikationsdauerFallNeu_StdAbw" + postfix, "PublikationsdauerFallNeu_Fallbasis" + postfix, "DatenstandTag_Max" + postfix] new_cases_delay.key = "DatenstandTag" print("new_cases_delay rows = {}, new_cases_to_count_delay = {}".format(new_cases_delay.nrows, new_cases_to_count_delay.nrows)) recovered_to_count = dayTable[(dt.f.NeuGenesen == 0) | (dt.f.NeuGenesen == 1),:] recovered = recovered_to_count[:, [dt.sum(dt.f.AnzahlGenesen)],dt.by(byDate)] recovered.names = ["DatenstandTag", "AnzahlGenesen"+postfix] recovered.key = "DatenstandTag" #print("recovered rows = {}".format(recovered.nrows)) new_recovered_to_count = dayTable[(dt.f.NeuGenesen == -1) | (dt.f.NeuGenesen == 1),:] new_recovered = new_recovered_to_count[:, [dt.sum(dt.f.AnzahlGenesen)],dt.by(byDate)] new_recovered.names = ["DatenstandTag", "AnzahlGenesenNeu"+postfix] new_recovered.key = "DatenstandTag" #print("new_recovered rows = {}".format(new_recovered.nrows)) byDayTable = byDayTable[:, :, dt.join(recovered)][:, :, dt.join(new_recovered)][:, :,dt.join(new_cases_delay)] #byDayTable = byDayTable[:,:,dt.join(recovered)][:,:,dt.join(new_recovered)]\ # [:,:,dt.join(new_cases_strict)][:,:,dt.join(new_cases_strict_14)][:,:,dt.join(new_cases_delay)] byDayTable.key = byDateColName #print("byDayTable rows = {}".format(byDayTable.nrows)) #print(byDayTable) return byDayTable