def transform(self, X: dt.Frame): logger = None if self.context and self.context.experiment_id: logger = make_experiment_logger( experiment_id=self.context.experiment_id, tmp_dir=self.context.tmp_dir, experiment_tmp_dir=self.context.experiment_tmp_dir) try: X = dt.Frame(X) original_zip_column_name = X.names[0] X.names = ['zip_key'] X = X[:, str('zip_key')] zip_list = dt.unique(X[~dt.isna(dt.f.zip_key), 0]).to_list()[0] zip_features = [self.get_zipcode_features(x) for x in zip_list] X_g = dt.Frame({"zip_key": zip_list}) X_g.cbind(dt.Frame(zip_features)) X_g.key = 'zip_key' X_result = X[:, :, dt.join(X_g)] self._output_feature_names = [ "{}.{}".format(original_zip_column_name, f) for f in list(X_result[:, 1:].names) ] self._feature_desc = [ "Property '{}' of US zipcode found in '{}'".format( f, original_zip_column_name) for f in list(X_result[:, 1:].names) ] return X_result[:, 1:] except Exception as ex: loggerwarning( logger, "USZipcodeDatabaseTransformer got exception {}".format( type(ex).__name__)) return np.zeros(X.shape[0])
def create_data( X: dt.Frame = None ) -> Union[str, List[str], dt.Frame, List[dt.Frame], np.ndarray, List[np.ndarray], pd.DataFrame, List[pd.DataFrame], Dict[ str, str], # {data set names : paths} Dict[str, dt.Frame], # {data set names : dt frames} Dict[str, np.ndarray], # {data set names : np arrays} Dict[str, pd.DataFrame], # {data set names : pd frames} ]: # define date column and forecast horizon date_col = 'date' forecast_len = 7 # get COVID19 new cases data from Our World in Data github X = dt.fread( "https://raw.githubusercontent.com/owid/covid-19-data/master/public/data/owid-covid-data.csv" ) # remove country aggregates like 'World' and 'International' X = X[~(dt.f.iso_code == '') & ~(dt.f.continent == ''), :] # determine threshold to split train and test based on forecast horizon dates = dt.unique(X[:, date_col]) split_date = dates[-(forecast_len + 1):, :, dt.sort(date_col)][0, 0] test_date = dates[-1, :, dt.sort(date_col)][0, 0] # split data to honor forecast horizon in test set train = X[dt.f[date_col] <= split_date, :] test = X[dt.f[date_col] > split_date, :] # return [train, test] and rename dataset names as needed return { f"covid19_daily_{split_date}_by_countries_train": train, f"covid19_daily_{test_date}_by_countries_test": test }
def test_10k_diabetes_xlsx(): filename = find_file("h2o-3", "fread", "10k_diabetes.xlsx") DT = dt.fread(filename) assert DT.shape == (10000, 51) assert DT.names[:4] == ("race", "gender", "age", "weight") assert DT["readmitted"].stype == dt.bool8 assert DT[:, "num_lab_procedures":"number_inpatient"].stype == dt.int32 assert dt.unique(DT["gender"]).nrows == 2
def create_data(X: dt.Frame = None) -> Union[ str, List[str], dt.Frame, List[dt.Frame], np.ndarray, List[np.ndarray], pd.DataFrame, List[pd.DataFrame], Dict[str, str], # {data set names : paths} Dict[str, dt.Frame], # {data set names : dt frames} Dict[str, np.ndarray], # {data set names : np arrays} Dict[str, pd.DataFrame], # {data set names : pd frames} ]: # define date column and forecast horizon date_col = 'date' forecast_len = 7 # get COVID19 data from NYTimes github us_total = dt.fread("https://raw.githubusercontent.com/nytimes/covid-19-data/master/us.csv") # produce lag of 1 unit and add as new feature for each column in the list series_cols = ["cases", "deaths"] aggs = {f"{col}_yesterday": shift(f[col]) for col in series_cols} us_total[:, update(**aggs), sort(date_col)] # update NA lags to 0 aggs = {f"{col}_yesterday": 0 for col in series_cols} us_total[isna(f[f"{series_cols[0]}_yesterday"]), update(**aggs)] # compute daily values by differentiating aggs = {f"{col}_daily": f[col] - f[f"{col}_yesterday"] for col in series_cols} us_total[:, update(**aggs), sort(date_col)] # delete columns with yesterday (shift) values series_cols_to_delete = [f"{col}_yesterday" for col in series_cols] del us_total[:, series_cols_to_delete] # set negative daily values to 0 us_total[f.cases_daily < 0, [f.cases_daily]] = 0 us_total[f.deaths_daily < 0, [f.deaths_daily]] = 0 # determine threshold to split train and test based on forecast horizon dates = dt.unique(us_total[:, date_col]) split_date = dates[-(forecast_len + 1):, :, dt.sort(date_col)][0, 0] test_date = dates[-1, :, dt.sort(date_col)][0, 0] # split data to honor forecast horizon in test set df = us_total[date_col].to_pandas() train = us_total[df[date_col] <= split_date, :] test = us_total[df[date_col] > split_date, :] # return [train, test] and rename dataset names as needed return {f"covid19_daily_{split_date}_us_train": train, f"covid19_daily_{test_date}_us_test": test}
def create_data( X: dt.Frame = None ) -> Union[str, List[str], dt.Frame, List[dt.Frame], np.ndarray, List[np.ndarray], pd.DataFrame, List[pd.DataFrame], Dict[ str, str], # {data set names : paths} Dict[str, dt.Frame], # {data set names : dt frames} Dict[str, np.ndarray], # {data set names : np arrays} Dict[str, pd.DataFrame], # {data set names : pd frames} ]: # define date column and forecast horizon date_col = 'date' forecast_len = 7 # get COVID19 data from NYTimes github us_states = dt.fread( "https://raw.githubusercontent.com/nytimes/covid-19-data/master/us-states.csv" ) # get states population us_states_pop = dt.fread( "http://www2.census.gov/programs-surveys/popest/datasets/2010-2019/national/totals/nst-est2019-alldata.csv" ) us_states_pop.names = {'NAME': 'state', 'POPESTIMATE2019': 'pop'} us_states_pop.key = "state" # augment data with state population figures and create adjusted case and death counts us_states[:, dt.update(pop=dt.g.pop, pop100k=dt.g.pop / 100000, cases100k=dt.f.cases / (dt.g.pop / 100000), deaths100k=dt.f.deaths / (dt.g.pop / 100000)), dt.join(us_states_pop)] # determine threshold to split train and test based on forecast horizon dates = dt.unique(us_states[:, date_col]) split_date = dates[-forecast_len:, :, dt.sort(date_col)][0, 0] # split data to honor forecast horizon in test set df = us_states[date_col].to_pandas() train = us_states[df[date_col] < split_date, :] test = us_states[df[date_col] >= split_date, :] # return [train, test] and rename dataset names as needed return { "covid19_daily_by_states_train": train, "covid10_daily_by_states_test": test }
def transform(self, X: dt.Frame): try: X = dt.Frame(X) X.names = ['zip_key'] X = X[:, str('zip_key')] zip_list = dt.unique(X[~dt.isna(dt.f.zip_key), 0]).to_list()[0] zip_features = [ self.get_zipcode_property(self.parse_zipcode(x)) for x in zip_list ] X_g = dt.Frame({ "zip_key": zip_list, self.get_property_name(): zip_features }) X_g.key = 'zip_key' X_result = X[:, :, dt.join(X_g)] return X_result[:, 1:] except: return np.zeros(X.shape[0])
def transform(self, X: dt.Frame): logger = None if self.context and self.context.experiment_id: logger = make_experiment_logger( experiment_id=self.context.experiment_id, tmp_dir=self.context.tmp_dir, experiment_tmp_dir=self.context.experiment_tmp_dir) X = dt.Frame(X) original_zip_column_name = X.names[0] X = X[:, dt.str64(dt.f[0])] X.names = ['zip_key'] try: zip_list = dt.unique(X[~dt.isna(dt.f.zip_key), 0]).to_list()[0] + ['79936'] zip_features = [self.get_zipcode_features(x) for x in zip_list] X_g = dt.Frame({"zip_key": zip_list}) X_g.cbind(dt.Frame(zip_features)) X_g.key = 'zip_key' X_result = X[:, :, dt.join(X_g)] self._output_feature_names = [ "{}:{}.{}".format(self.transformer_name, original_zip_column_name, self.replaceBannedCharacters(f)) for f in list(X_result[:, 1:].names) ] self._feature_desc = [ "Property '{}' of zipcode column ['{}'] from US zipcode database (recipe '{}')" .format(f, original_zip_column_name, self.transformer_name) for f in list(X_result[:, 1:].names) ] return X_result[:, 1:] except ValueError as ve: loggerinfo( logger, "Column '{}' is not a zipcode: {}".format( original_zip_column_name, str(ve))) return self.get_zipcode_null_result(X, original_zip_column_name) except TypeError as te: loggerwarning( logger, "Column '{}' triggered TypeError: {}".format( original_zip_column_name, str(te))) raise te
def transform(self, X: dt.Frame): X = dt.Frame(X) original_zip_column_name = X.names[0] X.names = ['zip_key'] X = X[:, str('zip_key')] zip_list = dt.unique(X[~dt.isna(dt.f.zip_key), 0]).to_list()[0] + ['79936'] zip_features = [self.get_zipcode_features(x) for x in zip_list] X_g = dt.Frame({"zip_key": zip_list}) X_g.cbind(dt.Frame(zip_features)) X_g.key = 'zip_key' X_result = X[:, :, dt.join(X_g)] self._output_feature_names = [ "{}.{}".format(original_zip_column_name, f) for f in list(X_result[:, 1:].names) ] self._feature_desc = [ "Property '{}' of US zipcode found in '{}'".format( f, original_zip_column_name) for f in list(X_result[:, 1:].names) ] return X_result[:, 1:]
def create_data(X: dt.Frame = None) -> Union[ str, List[str], dt.Frame, List[dt.Frame], np.ndarray, List[np.ndarray], pd.DataFrame, List[pd.DataFrame], Dict[str, str], # {data set names : paths} Dict[str, dt.Frame], # {data set names : dt frames} Dict[str, np.ndarray], # {data set names : np arrays} Dict[str, pd.DataFrame], # {data set names : pd frames} ]: # define date column and forecast horizon date_col = 'date' group_by_cols = ["state"] forecast_len = 7 # get COVID19 data from NYTimes github us_states = dt.fread("https://raw.githubusercontent.com/nytimes/covid-19-data/master/us-states.csv") # get states population us_states_pop = dt.fread( "http://www2.census.gov/programs-surveys/popest/datasets/2010-2019/national/totals/nst-est2019-alldata.csv") us_states_pop.names = {'NAME': 'state', 'POPESTIMATE2019': 'pop'} us_states_pop.key = "state" # augment data with state population figures and create adjusted case and death counts series_cols = ["cases", "deaths"] aggs = {f"{col}100k": dt.f[col] / (dt.g.pop / 100000) for col in series_cols} us_states[:, dt.update(pop = g.pop, pop100k = g.pop / 10000, **aggs), join(us_states_pop)] # remove rows without state defined (resulted in unmatched rows after left outer join) del us_states[isna(f.pop), :] # produce lag of 1 unit and add as new feature for each column in the list series_cols.extend([col + "100k" for col in series_cols]) aggs = {f"{col}_yesterday": shift(f[col]) for col in series_cols} us_states[:, update(**aggs), sort(date_col), by(group_by_cols)] # update NA lags to 0 aggs = {f"{col}_yesterday": 0 for col in series_cols} us_states[isna(f[f"{series_cols[0]}_yesterday"]), update(**aggs)] # compute daily values by differentiating aggs = {f"{col}_daily": f[col] - f[f"{col}_yesterday"] for col in series_cols} us_states[:, update(**aggs), sort(date_col), by(group_by_cols)] # delete columns with yesterday (shift) values series_cols_to_delete = [f"{col}_yesterday" for col in series_cols] del us_states[:, series_cols_to_delete] # set negative daily values to 0 us_states[f.cases_daily < 0, [f.cases_daily, f.cases100k_daily]] = 0 us_states[f.deaths_daily < 0, [f.deaths_daily, f.deaths100k_daily]] = 0 # determine threshold to split train and test based on forecast horizon dates = dt.unique(us_states[:, date_col]) split_date = dates[-(forecast_len + 1):, :, dt.sort(date_col)][0, 0] test_date = dates[-1, :, dt.sort(date_col)][0, 0] # split data to honor forecast horizon in test set df = us_states[date_col].to_pandas() train = us_states[df[date_col] <= split_date, :] test = us_states[df[date_col] > split_date, :] # return [train, test] and rename dataset names as needed return {f"covid19_daily_{split_date}_by_states_train": train, f"covid19_daily_{test_date}_by_states_test": test}
# Split dataset by partition id (column): results in as many partitions (datasets) # as there are values in parition column import datatable as dt # maximum number of partition to split allowed MAX_PARTITIONS = 10 # partition column name partition_col_name = 'quality' values = dt.unique(X[partition_col_name]).to_list()[0] if len(values) > MAX_PARTITIONS: raise ValueError("Too many partitions to split") result = {} for val in values: partition = X[dt.f[partition_col_name] == val, :] result.update({"mydata_" + str(val): partition}) return result
def analyze(fullTable, args): #fullTable = fullTable[dt.f.DatenstandTag > 382 - 20,:] print("Analyzing") pmu.printMemoryUsage("begin analyze") print("Keys:") print(fullTable.keys()) firstDumpDay = cint(fullTable[:, "DatenstandTag"].min()) lastDumpDay = cint(fullTable[:, "DatenstandTag"].max()) print("firstDumpDay", firstDumpDay) print("lastDumpDay", lastDumpDay) #fromDay = lastDumpDay-27 fromDay = firstDumpDay toDay = lastDumpDay + 1 fullTable = fullTable[:, dt.f[:].extend( {"MeldeDelay": dt.f.DatenstandTag - dt.f.MeldeTag - 1})] fullTable = fullTable[:, dt.f[:].extend( {"RefDelay": dt.f.DatenstandTag - dt.f.RefTag - 1})] fullTable.materialize() Altersgruppen = [] if args.agegroups: Altersgruppen = dt.unique(fullTable[:, "Altersgruppe"]).to_list()[0] print("Altersgruppen", Altersgruppen) Geschlechter = dt.unique(fullTable[:, "Geschlecht"]).to_list()[0] print("Geschlechter", Geschlechter) census = dt.fread("CensusByRKIAgeGroups.csv") censusDeutschland = census[dt.f.Name == "Deutschland", :] print(censusDeutschland) flaechen = loadFlaechen() #for id in range(1,16): # censusBL = census[dt.f.Code == id, :] # print(censusBL) print("Processing 'Deutschland'") pmu.printMemoryUsage("begin Deutschland") deutschland = analyzeDailyAltersgruppenGeschlechter( fullTable, filterByDay(fromDay, toDay), Altersgruppen, Geschlechter) deutschland = insertDates(deutschland) deutschland = insertRegionInfo(deutschland, 0, "Deutschland", "BR", 0, "Deutschland", flaechen[0]) print(deutschland) pmu.printMemoryUsage("pre makeIncidenceColumns") deutschland = makeIncidenceColumns(deutschland, censusDeutschland, Altersgruppen, Geschlechter) print(deutschland) pmu.printMemoryUsage("pre save") pmu.saveCsvTable(deutschland, "series-{}-{}.csv".format(0, "Deutschland"), args.outputDir) pmu.printMemoryUsage("post save") deutschland = None #exit(0) print("Processing Bundesländer") bundeslaender, bundeslaender_numbers = timeSeries(fullTable, fromDay, toDay, dt.f.IdBundesland, dt.f.Bundesland, Altersgruppen, Geschlechter) pmu.printMemoryUsage("post Bundesländer timeSeries") for i in range(bundeslaender.nrows): bl_name = bundeslaender[i, dt.f.Bundesland].to_list()[0][0] bl_id = bundeslaender[i, dt.f.IdBundesland].to_list()[0][0] if bl_id > 0: #bundeslaender_numbers[bl_id] = bundeslaender_numbers[bl_id][:, dt.f[:].extend( # {"IdLandkreis": bl_id, "Landkreis": bl_name, "IdBundesland": bl_id, "Bundesland": bl_name, "Flaeche" : flaechen[bl_id]})] bundeslaender_numbers[bl_id] = insertDates( bundeslaender_numbers[bl_id]) bundeslaender_numbers[bl_id] = insertRegionInfo( bundeslaender_numbers[bl_id], bl_id, bl_name, "BL", bl_id, bl_name, flaechen[0]) censusBL = census[dt.f.IdLandkreis == bl_id, :] print(censusBL) bundeslaender_numbers[bl_id] = makeIncidenceColumns( bundeslaender_numbers[bl_id], censusBL, Altersgruppen, Geschlechter) pmu.printMemoryUsage("pre save {}".format(bl_name)) pmu.saveCsvTable(bundeslaender_numbers[bl_id], "series-{}-{}.csv".format(bl_id, bl_name), args.outputDir) bundeslaender = None bundeslaender_numbers = None print("Processing Landkreise'") landKreise, landkreise_numbers = timeSeries(fullTable, fromDay, toDay, dt.f.IdLandkreis, dt.f.Landkreis, Altersgruppen, Geschlechter) pmu.printMemoryUsage("post Landkreise timeSeries") #print(landKreise) #print(landkreise_numbers) for i in range(landKreise.nrows): print(i) lk_name = landKreise[i, dt.f.Landkreis].to_list()[0][0] lk_id = landKreise[i, dt.f.IdLandkreis].to_list()[0][0] if lk_id > 0: censusLK = census[dt.f.IdLandkreis == lk_id, :] bl_name = censusLK[0, dt.f.Bundesland].to_list()[0][0] bl_id = censusLK[0, dt.f.IdBundesland].to_list()[0][0] lk_typ = landKreisTyp(lk_id, lk_name) landkreise_numbers[lk_id] = insertDates(landkreise_numbers[lk_id]) landkreise_numbers[lk_id] = insertRegionInfo( landkreise_numbers[lk_id], lk_id, lk_name, lk_typ, bl_id, bl_name, flaechen[lk_id]) #landkreise_numbers[lk_id] = landkreise_numbers[lk_id][:, dt.f[:].extend( # {"IdLandkreis": lk_id, "Landkreis": lk_name, "IdBundesland": bl_id, "Bundesland": bl_name, # "Flaeche": flaechen[lk_id]})] print(censusLK) landkreise_numbers[lk_id] = makeIncidenceColumns( landkreise_numbers[lk_id], censusLK, Altersgruppen, Geschlechter) pmu.printMemoryUsage("pre save {}".format(lk_name)) pmu.saveCsvTable(landkreise_numbers[lk_id], "series-{}-{}.csv".format(lk_id, lk_name), args.outputDir) #print(landKreise) return fullTable
def create_data( X: dt.Frame = None ) -> Union[str, List[str], dt.Frame, List[dt.Frame], np.ndarray, List[np.ndarray], pd.DataFrame, List[pd.DataFrame], Dict[ str, str], # {data set names : paths} Dict[str, dt.Frame], # {data set names : dt frames} Dict[str, np.ndarray], # {data set names : np arrays} Dict[str, pd.DataFrame], # {data set names : pd frames} ]: # define date column and forecast horizon date_col = 'date' group_by_cols = ["state"] forecast_len = 7 # state codes lookup table us_state_codes = dt.Frame( code=[ 'AL', 'AK', 'AS', 'AZ', 'AR', 'CA', 'CO', 'CT', 'DE', 'DC', 'FL', 'GA', 'GU', 'HI', 'ID', 'IL', 'IN', 'IA', 'KS', 'KY', 'LA', 'ME', 'MD', 'MA', 'MI', 'MN', 'MS', 'MO', 'MT', 'NE', 'NV', 'NH', 'NJ', 'NM', 'NY', 'NC', 'ND', 'MP', 'OH', 'OK', 'OR', 'PA', 'PR', 'RI', 'SC', 'SD', 'TN', 'TX', 'UT', 'VT', 'VI', 'VA', 'WA', 'WV', 'WI', 'WY' ], state=[ 'Alabama', 'Alaska', 'American Samoa', 'Arizona', 'Arkansas', 'California', 'Colorado', 'Connecticut', 'Delaware', 'District of Columbia', 'Florida', 'Georgia', 'Guam', 'Hawaii', 'Idaho', 'Illinois', 'Indiana', 'Iowa', 'Kansas', 'Kentucky', 'Louisiana', 'Maine', 'Maryland', 'Massachusetts', 'Michigan', 'Minnesota', 'Mississippi', 'Missouri', 'Montana', 'Nebraska', 'Nevada', 'New Hampshire', 'New Jersey', 'New Mexico', 'New York', 'North Carolina', 'North Dakota', 'Northern Mariana Islands', 'Ohio', 'Oklahoma', 'Oregon', 'Pennsylvania', 'Puerto Rico', 'Rhode Island', 'South Carolina', 'South Dakota', 'Tennessee', 'Texas', 'Utah', 'Vermont', 'Virgin Islands', 'Virginia', 'Washington', 'West Virginia', 'Wisconsin', 'Wyoming' ]) us_state_codes.key = "state" # get states population lookup table us_states_pop = dt.fread( "http://www2.census.gov/programs-surveys/popest/datasets/2010-2019/national/totals/nst-est2019-alldata.csv" ) us_states_pop.names = {'NAME': 'state', 'POPESTIMATE2019': 'pop'} us_states_pop = us_states_pop[dt.f.STATE > 0, :] us_states_pop.key = "state" # join state codes and population into single lookup table us_states_pop[:, dt.update(code=dt.g.code), dt.join(us_state_codes)] us_states_pop.key = "code" # US Covid Tracking API: https://covidtracking.com/data/api us_states = dt.fread( "https://covidtracking.com/api/v1/states/daily.csv") # remove deprecated fields deprecated = [ 'checkTimeEt', 'commercialScore', 'dateChecked', 'dateModified', 'grade', 'hash', 'hospitalized', 'negativeIncrease', 'negativeRegularScore', 'negativeScore', 'posNeg', 'positiveScore', 'score', 'total' ] us_states = us_states[:, list(set(us_states.names) - set(deprecated))] us_states.names = {'state': 'code'} series_cols = [ "positive", "negative", "hospitalizedCumulative", "inIcuCumulative", "onVentilatorCumulative", "recovered", "death" ] aggs = {f"{col}100k": f[col] / (g.pop / 100000) for col in series_cols} us_states[:, dt.update( state=g.state, pop=g.pop, pop100k=g.pop / 10000, **aggs), join(us_states_pop)] us_states = us_states[~dt.isna(dt.f.state), :] # produce lag of 1 unit and add as new feature for each shift column series_cols.extend([col + "100k" for col in series_cols]) aggs = {f"{col}_yesterday": shift(f[col]) for col in series_cols} us_states[:, update(**aggs), sort(date_col), by(group_by_cols)] # update NA lags aggs = {f"{col}_yesterday": 0 for col in series_cols} us_states[isna(f[f"{series_cols[0]}_yesterday"]), update(**aggs)] aggs = { f"{col}_daily": f[col] - f[f"{col}_yesterday"] for col in series_cols } us_states[:, update(**aggs), sort(date_col), by(group_by_cols)] for col in series_cols: del us_states[:, f[f"{col}_yesterday"]] # validate dataset if us_states[:, count(), by(dt.f.state, f.date)][f.count > 1, :].shape[0] > 1: raise ValueError( "Found duplicate elements for the same date and state.") # determine threshold to split train and test based on forecast horizon dates = dt.unique(us_states[:, date_col]) split_date = dates[-(forecast_len + 1):, :, dt.sort(date_col)][0, 0] test_date = dates[-1, :, dt.sort(date_col)][0, 0] # split data to honor forecast horizon in test set df = us_states[date_col].to_pandas() train = us_states[df[date_col] <= split_date, :] test = us_states[df[date_col] > split_date, :] return { f"covidtracking_daily_{split_date}_by_us_states_train": train, f"covidtracking_daily_{test_date}_by_us_states_test": test }
def analyze(fullTable, args, oldTables): #fullTable = fullTable[dt.f.DatenstandTag <= 387,:] print("Analyzing") pmu.printMemoryUsage("begin analyze") print("Keys:") print(fullTable.keys()) print(list(zip(fullTable.names, fullTable.stypes))) daysInfullTable = dt.unique(fullTable[:, "DatenstandTag"]).to_list()[0] firstDumpDay = min(daysInfullTable) lastDumpDay = max(daysInfullTable) maxMeldeDay = cint(fullTable[:,"MeldeTag"].max()) if maxMeldeDay > lastDumpDay: print("Future Date in Meldetag ({}), clipping to yesterday, Datenstandtag-1 = {}".format(maxMeldeDay, lastDumpDay)) fullTable["MeldeTag">=lastDumpDay,"MeldeTag"] = lastDumpDay -1 print("firstDumpDay", firstDumpDay) print("lastDumpDay",lastDumpDay) print("maxMeldeDay",maxMeldeDay) fromDay = firstDumpDay toDay = lastDumpDay+1 #fromDay = lastDumpDay-1 if len(oldTables)>0: # calculate which rows are needed for the update daysInOldTables = dt.unique(oldTables[0][:, "DatenstandTag"]).to_list()[0] newDays = sorted(list(set(daysInfullTable).difference(set(daysInOldTables)))) print("newDays",newDays) if len(newDays) == 0: print("Nothing to update") exit(9) minNewDay = min(newDays) maxNewDay = max(newDays) minNewDay7daysAgo = minNewDay - 7 maxNewDay7daysAgo = maxNewDay - 7 fullTable = fullTable[((dt.f.DatenstandTag >= minNewDay) & (dt.f.DatenstandTag <= maxNewDay)) | ((dt.f.DatenstandTag >= minNewDay7daysAgo) & (dt.f.DatenstandTag <= maxNewDay7daysAgo)),:] #fullTable.materialize() daysInfullTable = dt.unique(fullTable[:, "DatenstandTag"]).to_list()[0] print("daysInfullTable",daysInfullTable) fullTable = fullTable[:, dt.f[:].extend({"MeldeDelay": dt.f.DatenstandTag-dt.f.MeldeTag-1})] fullTable = fullTable[:, dt.f[:].extend({"RefDelay": dt.f.DatenstandTag-dt.f.RefTag-1})] #fullTable.materialize() Altersgruppen = [] if args.agegroups: Altersgruppen = dt.unique(fullTable[:,"Altersgruppe"]).to_list()[0] print("Altersgruppen", Altersgruppen) Geschlechter = [] if args.gender: Geschlechter = dt.unique(fullTable[:,"Geschlecht"]).to_list()[0] print("Geschlechter", Geschlechter) census = dt.fread("CensusByRKIAgeGroups.csv") censusDeutschland = census[dt.f.Name == "Deutschland",:] print(censusDeutschland) flaechen = loadFlaechen() print("Processing 'Deutschland'") pmu.printMemoryUsage("begin Deutschland") deutschland = analyzeDailyAltersgruppenGeschlechter(fullTable, fromDay, toDay, True, True, Altersgruppen, Geschlechter) deutschland = insertDates(deutschland) deutschland = insertRegionInfo(deutschland, 0, "Deutschland", "BR", 0, "Deutschland", flaechen[0]) deutschland = insertEinwohnerColumns(deutschland, censusDeutschland, Altersgruppen, Geschlechter, "Flaeche") print(deutschland) pmu.printMemoryUsage("pre makeIncidenceColumns") #deutschland = makeIncidenceColumns(deutschland, censusDeutschland, Altersgruppen, Geschlechter) #print(deutschland) if len(oldTables) > 0: deutschland = updateOldTable(oldTables[0], deutschland) pmu.printMemoryUsage("pre save") pmu.saveCsvTable(deutschland, "series-{}-{}.csv".format(0, "Deutschland"), args.outputDir) pmu.printMemoryUsage("post save") deutschland = None #exit(0) print("Processing Bundesländer") bundeslaender, bundeslaender_numbers = timeSeries(fullTable, fromDay, toDay, dt.f.IdBundesland, dt.f.Bundesland, Altersgruppen, Geschlechter) pmu.printMemoryUsage("post Bundesländer timeSeries") for i in range(bundeslaender.nrows): bl_name=bundeslaender[i,dt.f.Bundesland].to_list()[0][0] bl_id=bundeslaender[i,dt.f.IdBundesland].to_list()[0][0] if bl_id > 0: bundeslaender_numbers[bl_id] = insertDates(bundeslaender_numbers[bl_id]) bundeslaender_numbers[bl_id] = insertRegionInfo(bundeslaender_numbers[bl_id], bl_id, bl_name, "BL", bl_id, bl_name, flaechen[0]) censusBL = census[dt.f.IdLandkreis == bl_id, :] bundeslaender_numbers[bl_id] = insertEinwohnerColumns(bundeslaender_numbers[bl_id], censusBL, Altersgruppen, Geschlechter, "Flaeche") if len(oldTables) > 0: bundeslaender_numbers[bl_id] = updateOldTable(oldTables[bl_id], bundeslaender_numbers[bl_id]) print(censusBL) pmu.printMemoryUsage("pre save {}".format(bl_name)) pmu.saveCsvTable(bundeslaender_numbers[bl_id], "series-{}-{}.csv".format(bl_id, bl_name), args.outputDir) bundeslaender = None bundeslaender_numbers = None print("Processing Landkreise'") landKreise, landkreise_numbers = timeSeries(fullTable, fromDay, toDay, dt.f.IdLandkreis, dt.f.Landkreis, Altersgruppen, Geschlechter) pmu.printMemoryUsage("post Landkreise timeSeries") #print(landKreise) #print(landkreise_numbers) for i in range(landKreise.nrows): print(i) lk_name = landKreise[i, dt.f.Landkreis].to_list()[0][0] lk_id = landKreise[i, dt.f.IdLandkreis].to_list()[0][0] if lk_name == "LK Saarpfalz-Kreis": lk_name = "LK Saar-Pfalz-Kreis" if lk_id > 0: censusLK = census[dt.f.IdLandkreis == lk_id, :] bl_name = censusLK[0,dt.f.Bundesland].to_list()[0][0] bl_id = censusLK[0, dt.f.IdBundesland].to_list()[0][0] lk_typ = landKreisTyp(lk_id, lk_name) landkreise_numbers[lk_id] = insertDates(landkreise_numbers[lk_id]) landkreise_numbers[lk_id] = insertRegionInfo(landkreise_numbers[lk_id], lk_id, lk_name, lk_typ, bl_id, bl_name, flaechen[lk_id]) #print(censusLK) landkreise_numbers[lk_id] = insertEinwohnerColumns(landkreise_numbers[lk_id], censusLK, Altersgruppen, Geschlechter, "Flaeche") if len(oldTables) > 0: landkreise_numbers[lk_id] = updateOldTable(oldTables[lk_id], landkreise_numbers[lk_id]) pmu.printMemoryUsage("pre save {}".format(lk_name)) pmu.saveCsvTable(landkreise_numbers[lk_id], "series-{}-{}.csv".format(lk_id, lk_name), args.outputDir) #print(landKreise) return fullTable