def test_sd():
assert str(dt.sd(f.A)) == str(f.A.sd())
assert str(dt.sd(f[:])) == str(f[:].sd())
DT = dt.Frame({"C": [2, 5, 30, 20, 10],
"D": [10, 8, 20, 20, 1]})
assert_equals(DT[:, f[:].sd()], DT[:, dt.sd(f[:])])
def test_rows_stdev():
from datatable import sd
df0 = dt.Frame(range(10), names=["A"])
# stdev = 3.0276
df1 = df0(f.A > sd(f.A), engine="eager")
assert df1.internal.check()
assert df1.topython() == [[4, 5, 6, 7, 8, 9]]
def test_rows_stdev():
from datatable import sd
df0 = dt.Frame(A=range(10))
# stdev = 3.0276
df1 = df0[f.A > sd(f.A), :]
frame_integrity_check(df1)
assert df1.to_list() == [[4, 5, 6, 7, 8, 9]]
fun=fun,
run=2,
time_sec=t,
mem_gb=m,
cache=cache,
chk=make_chk(flatten(chk.to_list())),
chk_time_sec=chkt,
on_disk=on_disk)
print(ans.head(3), flush=True)
print(ans.tail(3), flush=True)
del ans
question = 'median v3 sd v3 by id4 id5' # q6
gc.collect()
t_start = timeit.default_timer()
ans = x[:, {'median_v3': median(f.v3), 'sd_v3': sd(f.v3)}, by(f.id4, f.id5)]
print(ans.shape, flush=True)
t = timeit.default_timer() - t_start
m = memory_usage()
t_start = timeit.default_timer()
chk = ans[:, [sum(f.median_v3), sum(f.sd_v3)]]
chkt = timeit.default_timer() - t_start
write_log(task=task,
data=data_name,
in_rows=x.shape[0],
question=question,
out_rows=ans.shape[0],
out_cols=ans.shape[1],
solution=solution,
version=ver,
git=git,
def analyzeDaily(fullTable, filter, postfix):
#print("----- analyzeDaily:"+postfix)
#dayTable = fullTable[(dt.f.DatenstandTag >= fromDay) & (dt.f.DatenstandTag < toDay) & (dt.f.IdLandkreis == forIdLandkreis),:]
dayTable = fullTable[filter, :]
cases_to_count = dayTable[(dt.f.NeuerFall == 0) | (dt.f.NeuerFall == 1), :]
cases = cases_to_count[:, [dt.sum(dt.f.AnzahlFall)],
dt.by(dt.f.DatenstandTag)]
cases.names = ["DatenstandTag", "AnzahlFall" + postfix]
cases.key = "DatenstandTag"
print("cases rows = {}, cases_to_count = {}".format(
cases.nrows, cases_to_count.nrows))
#print(cases)
new_cases_to_count = dayTable[(dt.f.NeuerFall == -1) |
(dt.f.NeuerFall == 1), :]
new_cases = new_cases_to_count[:, [dt.sum(dt.f.AnzahlFall)],
dt.by(dt.f.DatenstandTag)]
new_cases.names = ["DatenstandTag", "AnzahlFallNeu" + postfix]
new_cases.key = "DatenstandTag"
print("new_cases rows = {}, new_cases_to_count = {}".format(
new_cases.nrows, new_cases_to_count.nrows))
#new_cases_to_count.to_csv("new_cases_to_count.csv")
new_cases_to_count_delay = new_cases_to_count[(
dt.f.AnzahlFall > 0), :] # measure delay only for positive cases
new_cases_to_count_delay.materialize()
new_cases_delay = new_cases_to_count_delay[:, [
dt.min(dt.f.MeldeDelay),
dt.max(dt.f.MeldeDelay),
dt.mean(dt.f.MeldeDelay),
dt.median(dt.f.MeldeDelay),
dt.sd(dt.f.MeldeDelay),
dt.sum(dt.f.AnzahlFall),
dt.max(dt.f.DatenstandTag)
],
dt.by(dt.f.DatenstandTag)]
new_cases_delay.names = [
"DatenstandTag", "MeldeDauerFallNeu-Min" + postfix,
"MeldeDauerFallNeu-Max" + postfix,
"MeldeDauerFallNeu-Schnitt" + postfix,
"MeldeDauerFallNeu-Median" + postfix,
"MeldeDauerFallNeu-StdAbw" + postfix,
"MeldeDauerFallNeu-Fallbasis" + postfix, "DatenstandTag-Max" + postfix
]
new_cases_delay.key = "DatenstandTag"
print("new_cases_delay rows = {}, new_cases_to_count_delay = {}".format(
new_cases_delay.nrows, new_cases_to_count_delay.nrows))
#new_cases_delay = new_cases_to_count_delay[:, [dt.mean(dt.f.DatenstandTag-dt.f.MeldeTag)],dt.by(dt.f.DatenstandTag)]
# delays = delayRecs[:, [dt.mean(dt.f.MeldeDelay), dt.median(dt.f.MeldeDelay), dt.sd(dt.f.MeldeDelay), dt.sum(dt.f.AnzahlFall)], dt.by(dt.f.Landkreis)]
# new_cases_stddev = new_cases_to_count_delay[:, [dt.mean(dt.f.DatenstandTag - dt.f.MeldeTag)],
# dt.by(dt.f.DatenstandTag)]
# new_cases_delay.names = ["DatenstandTag", "AnzahlFallNeu-MeldeDauer" + postfix]
# new_cases_delay.key = "DatenstandTag"
# print("new_cases_delay rows = {}, new_cases_to_count_delay = {}".format(new_cases_delay.nrows,
# new_cases_to_count_delay.nrows))
new_cases_to_count_strict = new_cases_to_count[(
dt.f.DatenstandTag - dt.f.MeldeTag < 7) | (dt.f.AnzahlFall < 0), :]
new_cases_strict = new_cases_to_count_strict[:, [dt.sum(dt.f.AnzahlFall)],
dt.by(dt.f.DatenstandTag)]
new_cases_strict.names = [
"DatenstandTag", "AnzahlFallNeu-Meldung-letze-7-Tage" + postfix
]
new_cases_strict.key = "DatenstandTag"
print("new_cases_strict rows = {}, new_cases_to_count_strict = {}".format(
new_cases_strict.nrows, new_cases_to_count_strict.nrows))
#new_cases_to_count_strict.to_csv("new_cases_to_count_strict.csv")
new_cases_to_count_strict_14 = new_cases_to_count[(
dt.f.DatenstandTag - dt.f.MeldeTag < 14) | (dt.f.AnzahlFall < 0), :]
new_cases_strict_14 = new_cases_to_count_strict_14[:, [
dt.sum(dt.f.AnzahlFall)
], dt.by(dt.f.DatenstandTag)]
new_cases_strict_14.names = [
"DatenstandTag", "AnzahlFallNeu-Meldung-letze-14-Tage" + postfix
]
new_cases_strict_14.key = "DatenstandTag"
print("new_cases_strict_14 rows = {}, new_cases_to_count_strict_14 = {}".
format(new_cases_strict_14.nrows,
new_cases_to_count_strict_14.nrows))
dead_to_count = dayTable[(dt.f.NeuerTodesfall == 0) |
(dt.f.NeuerTodesfall == 1), :]
dead = dead_to_count[:, [dt.sum(dt.f.AnzahlTodesfall)],
dt.by(dt.f.DatenstandTag)]
dead.names = ["DatenstandTag", "AnzahlTodesfall" + postfix]
dead.key = "DatenstandTag"
#print("dead rows = {}".format(dead.nrows))
new_dead_to_count = dayTable[(dt.f.NeuerTodesfall == -1) |
(dt.f.NeuerTodesfall == 1), :]
new_dead = new_dead_to_count[:, [dt.sum(dt.f.AnzahlTodesfall)],
dt.by(dt.f.DatenstandTag)]
new_dead.names = ["DatenstandTag", "AnzahlTodesfallNeu" + postfix]
new_dead.key = "DatenstandTag"
#print("new_dead rows = {}".format(new_dead.nrows))
recovered_to_count = dayTable[(dt.f.NeuGenesen == 0) |
(dt.f.NeuGenesen == 1), :]
recovered = recovered_to_count[:, [dt.sum(dt.f.AnzahlGenesen)],
dt.by(dt.f.DatenstandTag)]
recovered.names = ["DatenstandTag", "AnzahlGenesen" + postfix]
recovered.key = "DatenstandTag"
#print("recovered rows = {}".format(recovered.nrows))
new_recovered_to_count = dayTable[(dt.f.NeuGenesen == -1) |
(dt.f.NeuGenesen == 1), :]
new_recovered = new_recovered_to_count[:, [dt.sum(dt.f.AnzahlGenesen)],
dt.by(dt.f.DatenstandTag)]
new_recovered.names = ["DatenstandTag", "AnzahlGenesenNeu" + postfix]
new_recovered.key = "DatenstandTag"
#print("new_recovered rows = {}".format(new_recovered.nrows))
byDayTable = cases[:,:,dt.join(new_cases)]\
[:,:,dt.join(dead)][:,:,dt.join(new_dead)][:,:,dt.join(recovered)][:,:,dt.join(new_recovered)]\
[:,:,dt.join(new_cases_strict)][:,:,dt.join(new_cases_strict_14)][:,:,dt.join(new_cases_delay)]
byDayTable.key = "DatenstandTag"
#print("byDayTable rows = {}".format(byDayTable.nrows))
print(byDayTable)
return byDayTable
def analyzeDaily(fullTable, filter, prefix, postfix, byDateColName):
print("analyzeDaily prefix='{}' postfix='{}' byDateColName='{}'".format(prefix, postfix, byDateColName))
#print("analyzeDaily filter='{}' '".format(filter))
byDate = dt.f[byDateColName]
#print("----- analyzeDaily:"+postfix)
#dayTable = fullTable[(dt.f.DatenstandTag >= fromDay) & (dt.f.DatenstandTag < toDay) & (dt.f.IdLandkreis == forIdLandkreis),:]
dayTable = fullTable[filter,:]
cases_to_count = dayTable[(dt.f.NeuerFall == 0) | (dt.f.NeuerFall == 1),:]
cases = cases_to_count[:, [dt.sum(dt.f.AnzahlFall)],dt.by(byDate)]
cases.names = [byDateColName, prefix+"AnzahlFall"+postfix]
cases.key = byDateColName
print("cases rows = {}, cases_to_count = {}".format(cases.nrows, cases_to_count.nrows))
#print(cases)
byDayTable = cases
if byDateColName == "DatenstandTag":
new_cases_to_count = dayTable[(dt.f.NeuerFall == -1) | (dt.f.NeuerFall == 1),:]
new_cases = new_cases_to_count[:, [dt.sum(dt.f.AnzahlFall)],dt.by(byDate)]
new_cases.names = [byDateColName, prefix+"AnzahlFallNeu"+postfix]
new_cases.key = byDateColName
print("new_cases rows = {}, new_cases_to_count = {}".format(new_cases.nrows, new_cases_to_count.nrows))
#new_cases_to_count.to_csv("new_cases_to_count.csv")
byDayTable = byDayTable[:,:,dt.join(new_cases)]
else:
# add days by MeldeTag
byDayTable.names = {prefix+"AnzahlFall"+postfix: prefix+"AnzahlFallNeu"+postfix}
byDayTable = addRunningSumColumn(byDayTable, prefix+"AnzahlFallNeu"+postfix, prefix+"AnzahlFall"+postfix)
dead_to_count = dayTable[(dt.f.NeuerTodesfall == 0) | (dt.f.NeuerTodesfall == 1),:]
dead = dead_to_count[:, [dt.sum(dt.f.AnzahlTodesfall)],dt.by(byDate)]
dead.names = [byDateColName, prefix+"AnzahlTodesfall"+postfix]
dead.key = byDateColName
#print("dead rows = {}".format(dead.nrows))
byDayTable = byDayTable[:,:,dt.join(dead)]
if byDateColName == "DatenstandTag":
new_dead_to_count = dayTable[(dt.f.NeuerTodesfall == -1) | (dt.f.NeuerTodesfall == 1),:]
new_dead = new_dead_to_count[:, [dt.sum(dt.f.AnzahlTodesfall)],dt.by(byDate)]
new_dead.names = [byDateColName, prefix+"AnzahlTodesfallNeu"+postfix]
new_dead.key = byDateColName
#print("new_dead rows = {}".format(new_dead.nrows))
byDayTable = byDayTable[:,:,dt.join(new_dead)]
else:
# add days by MeldeTag
byDayTable.names = {prefix+"AnzahlTodesfall"+postfix: prefix+"AnzahlTodesfallNeu"+postfix}
byDayTable = addRunningSumColumn(byDayTable, prefix+"AnzahlTodesfallNeu"+postfix, prefix+"AnzahlTodesfall"+postfix)
byDayTable.key = byDateColName
if postfix == "" and prefix == "" and byDateColName == "DatenstandTag":
new_cases_to_count_delay = new_cases_to_count[(dt.f.AnzahlFall > 0), :] # measure delay only for positive cases
new_cases_to_count_delay.materialize()
new_cases_delay = new_cases_to_count_delay[:, [dt.min(dt.f.MeldeDelay), dt.max(dt.f.MeldeDelay),
dt.mean(dt.f.MeldeDelay), dt.median(dt.f.MeldeDelay),
dt.sd(dt.f.MeldeDelay), dt.sum(dt.f.AnzahlFall),
dt.max(dt.f.DatenstandTag)], dt.by(byDate)]
new_cases_delay.names = ["DatenstandTag",
"PublikationsdauerFallNeu_Min" + postfix, "PublikationsdauerFallNeu_Max" + postfix,
"PublikationsdauerFallNeu_Schnitt" + postfix, "PublikationsdauerFallNeu_Median" + postfix,
"PublikationsdauerFallNeu_StdAbw" + postfix, "PublikationsdauerFallNeu_Fallbasis" + postfix,
"DatenstandTag_Max" + postfix]
new_cases_delay.key = "DatenstandTag"
print("new_cases_delay rows = {}, new_cases_to_count_delay = {}".format(new_cases_delay.nrows,
new_cases_to_count_delay.nrows))
recovered_to_count = dayTable[(dt.f.NeuGenesen == 0) | (dt.f.NeuGenesen == 1),:]
recovered = recovered_to_count[:, [dt.sum(dt.f.AnzahlGenesen)],dt.by(byDate)]
recovered.names = ["DatenstandTag", "AnzahlGenesen"+postfix]
recovered.key = "DatenstandTag"
#print("recovered rows = {}".format(recovered.nrows))
new_recovered_to_count = dayTable[(dt.f.NeuGenesen == -1) | (dt.f.NeuGenesen == 1),:]
new_recovered = new_recovered_to_count[:, [dt.sum(dt.f.AnzahlGenesen)],dt.by(byDate)]
new_recovered.names = ["DatenstandTag", "AnzahlGenesenNeu"+postfix]
new_recovered.key = "DatenstandTag"
#print("new_recovered rows = {}".format(new_recovered.nrows))
byDayTable = byDayTable[:, :, dt.join(recovered)][:, :, dt.join(new_recovered)][:, :,dt.join(new_cases_delay)]
#byDayTable = byDayTable[:,:,dt.join(recovered)][:,:,dt.join(new_recovered)]\
# [:,:,dt.join(new_cases_strict)][:,:,dt.join(new_cases_strict_14)][:,:,dt.join(new_cases_delay)]
byDayTable.key = byDateColName
#print("byDayTable rows = {}".format(byDayTable.nrows))
#print(byDayTable)
return byDayTable
