(self.df['nuts_3'] == 'Total')].index,
inplace=True)
self.df.replace("Wales Total", "Wales", inplace=True)
if __name__ == "__main__":
column_converter = {"authority": "nuts_3"}
daily_files = retrieve_files(WALES_DAILY_FOLDER)
daily_files.sort()
for file in daily_files:
file_path = os.path.join(WALES_DAILY_FOLDER, file)
file_transformation = DailyTransformation(
file_path=file_path, column_converter=column_converter)
file_transformation.workflow()
cov_wales = SARSCOV2Wales()
cov_wales.workflow()
logger.info(cov_wales.df)
da_wales = DailyAggregator(base_folder="dataset",
daily_folder=WALES_DAILY_FOLDER,
country="Wales")
da_wales.workflow()
print("End of Game")
"""Get datetime of dataset
"""
# Last updated: 2pm on 16 March 2020
re_dt = re.compile(r'Updated: (.*) (.*) (.*)</em></p>')
dt_from_re = re_dt.findall(self.req.text)
if not dt_from_re:
raise Exception("Did not find datetime from webpage")
dt_from_re = dt_from_re[0]
dt_from_re = " ".join(dt_from_re)
dt_from_re = dateutil.parser.parse(dt_from_re, dayfirst=True)
self.dt = dt_from_re
def post_processing(self):
self.df.sort_values(by="cases", inplace=True)
if __name__ == "__main__":
cov_england = SARSCOV2England()
cov_england.workflow()
da_england = DailyAggregator(base_folder="dataset",
daily_folder=ENGLAND_DAILY_FOLDER,
country="England")
da_england.workflow()
print("End of Game")
self.df.rename(columns={
"Gemeente": "city",
"Aantal": "cases",
"BevAant": "population",
"Aantal per 100.000 inwoners": "cases/100k pop."
},
inplace=True)
self.df = self.df[[
"country", "city", "cases", "population", "cases/100k pop.",
"datetime"
]]
self.df.sort_values(by="cases", inplace=True)
if __name__ == "__main__":
cov_nl = SARSCOV2NL()
cov_nl.workflow()
print(cov_nl.df)
if cov_nl.df.empty:
raise Exception("Empty dataframe for NL data")
da = DailyAggregator(base_folder="dataset",
daily_folder=DAILY_FOLDER,
country="NL")
da.workflow()
print("End of Game")
# re_dt = re.compile(r'as of (.*)')
re_dt = re.compile(r'as of\s+(\d+\s\w+\s\d+)')
re_dt_res = re_dt.findall(el[0].xpath('.//h1/span/text()')[0])
if not re_dt_res:
raise Exception("Could not find datetime on the web page")
# self.dt = dateutil.parser.parse(re_dt_res[0], dayfirst=True)
# self.dt = datetime.datetime.strptime(re_dt_res[0] + " 1", "week %W %Y %w")
self.dt = pd.to_datetime(re_dt_res[0])
def add_country_to_df(self):
logger.debug("No need to add country")
def post_processing(self):
self.df.sort_values(by="cases", inplace=True)
if __name__ == "__main__":
cov_ecdc = SARSCOV2ECDC()
cov_ecdc.workflow()
da_ecdc = DailyAggregator(base_folder="dataset",
daily_folder=DAILY_FOLDER,
country="ECDC")
da_ecdc.workflow()
print("End of Game")
# drop_rows = {
# "state": "sum"
# }
# daily_files = retrieve_files(DAILY_FOLDER)
# daily_files.sort()
# for file in daily_files:
# file_path = os.path.join(DAILY_FOLDER, file)
# file_transformation = DailyTransformation(
# file_path=file_path,
# column_converter=column_converter,
# drop_rows=drop_rows
# )
# file_transformation.workflow()
cov_at = SARSCOV2AT()
cov_at.workflow()
print(cov_at.df)
da = DailyAggregator(
base_folder="dataset",
daily_folder=DAILY_FOLDER,
country="AT",
fill=False
)
da.workflow()
print("End of Game")
# daily_files.sort()
# for file in daily_files:
# file_path = os.path.join(DAILY_FOLDER, file)
# file_transformation = DailyTransformation(
# file_path=file_path,
# column_converter=column_converter,
# drop_rows=drop_rows
# )
# file_transformation.workflow()
cov_nl = SARSCOV2NL()
cov_nl.workflow()
print(cov_nl.df)
if cov_nl.df.empty:
raise Exception("Empty dataframe for NL data")
da = DailyAggregator(base_folder="dataset",
daily_folder=DAILY_FOLDER,
country="NL",
replace={
"Noardeast-Fryslân": "Noardeast-Fryslân",
"Súdwest-Fryslân": "Súdwest-Fryslân",
"Súdwest Fryslân": "Súdwest-Fryslân",
"s-Gravenhage": "'s-Gravenhage"
})
da.workflow()
print("End of Game")
if __name__ == "__main__":
column_converter = {
"nuts_2": "nuts_3"
}
daily_files = retrieve_files(SCOTLAND_DAILY_FOLDER)
daily_files.sort()
for file in daily_files:
file_path = os.path.join(SCOTLAND_DAILY_FOLDER, file)
file_transformation = DailyTransformation(
file_path=file_path,
column_converter=column_converter
)
file_transformation.workflow()
cov_scotland = SARSCOV2Scotland()
cov_scotland.workflow()
da_scotland = DailyAggregator(
base_folder="dataset",
daily_folder=SCOTLAND_DAILY_FOLDER,
country="Scotland"
)
da_scotland.workflow()
print("End of Game")