if self.df.equals(df_existing): exists = True logger.info("Current data frame already exists") break if not exists: self.df.to_csv( f"{self.daily_folder}/{self.country.lower()}_covid19_{self.date}_{self.hour:0.0f}_{self.minute:02.0f}.csv", index=False) if __name__ == "__main__": column_converter = {"authority": "nuts_3"} daily_files = retrieve_files(ENGLAND_DAILY_FOLDER) daily_files.sort() for file in daily_files: file_path = os.path.join(ENGLAND_DAILY_FOLDER, file) file_transformation = DailyTransformation( file_path=file_path, column_converter=column_converter) file_transformation.workflow() cov_england = SARSCOV2England() cov_england.workflow() da_england = DailyAggregator(base_folder="dataset", daily_folder=ENGLAND_DAILY_FOLDER, country="England") da_england.workflow()
def post_processing(self): self.df.sort_values(by="cases", inplace=True) self.df.drop(self.df.loc[(self.df['nuts_3'] == 'TOTAL') | (self.df['nuts_3'] == 'Total')].index, inplace=True) self.df.replace("Wales Total", "Wales", inplace=True) if __name__ == "__main__": column_converter = {"authority": "nuts_3"} daily_files = retrieve_files(WALES_DAILY_FOLDER) daily_files.sort() for file in daily_files: file_path = os.path.join(WALES_DAILY_FOLDER, file) file_transformation = DailyTransformation( file_path=file_path, column_converter=column_converter) file_transformation.workflow() cov_wales = SARSCOV2Wales() cov_wales.workflow() logger.info(cov_wales.df) da_wales = DailyAggregator(base_folder="dataset", daily_folder=WALES_DAILY_FOLDER,
def post_processing(self): self.df.sort_values(by="cases", inplace=True) if __name__ == "__main__": column_converter = { "state": "nuts_2" } drop_rows = { "state": "sum" } daily_files = retrieve_files(DAILY_FOLDER) daily_files.sort() for file in daily_files: file_path = os.path.join(DAILY_FOLDER, file) file_transformation = DailyTransformation( file_path=file_path, column_converter=column_converter, drop_rows=drop_rows ) file_transformation.workflow() cov_at = SARSCOV2AT() cov_at.workflow() print(cov_at.df)
dt_from_re = dt_from_re[0] dt_from_re = dateutil.parser.parse(dt_from_re, dayfirst=True) self.dt = dt_from_re def post_processing(self): self.df.sort_values(by="cases", inplace=True) if __name__ == "__main__": column_converter = { "nuts_2": "nuts_3" } daily_files = retrieve_files(SCOTLAND_DAILY_FOLDER) daily_files.sort() for file in daily_files: file_path = os.path.join(SCOTLAND_DAILY_FOLDER, file) file_transformation = DailyTransformation( file_path=file_path, column_converter=column_converter ) file_transformation.workflow() cov_scotland = SARSCOV2Scotland() cov_scotland.workflow() da_scotland = DailyAggregator( base_folder="dataset",