def check_for_new_data():
log.info("Checking for new data...")
try:
req = requests.get(config.NATIONAL_DATA_JSON_URL)
except RequestException as req:
log.error("Error occurred while requesting data: " + str(req))
return
if req.status_code == 200:
try:
dp = DataProcessor.initialize(req.content, config.DATE_FORMAT)
except InvalidDataFormatException as err:
log.error("Received invalid data: " + str(err))
return
last_data_date = dp.get("date", start=dp.size() - 1)[0]
last_exec_date = read_last_date_updated(
config.LATEST_EXECUTION_DATE_FILE_PATH)
if last_exec_date is None or last_data_date > last_exec_date or DEBUG_MODE:
log.info("New data found, processing and tweeting...")
dp.localize_dates("UTC", "Europe/Rome")
charts_paths = generate_graphs(dp)
tweet_updates(dp, charts_paths)
if not DEBUG_MODE:
write_last_date_updated(config.LATEST_EXECUTION_DATE_FILE_PATH,
last_data_date)
log.info("New data tweeted successfully.")
else:
log.info("No updates found.")
else:
log.warning("Got {0} status code.".format(req.status_code))
def test_create_data_processor_with_wrong_date_type_format(self):
with self.assertRaises(InvalidDataFormatException):
DataProcessor.initialize([{
"data": "2020-02-28",
"ricoverati_con_sintomi": "345",
"terapia_intensiva": 64,
"totale_ospedalizzati": 409,
"isolamento_domiciliare": 412,
"totale_positivi": 821,
"variazione_totale_positivi": 233,
"nuovi_positivi": 238,
"dimessi_guariti": 46,
"deceduti": 21,
"totale_casi": 888,
"tamponi": 15695,
}], config.DATE_FORMAT)
def test_get_all_values_of_existing_property(self):
dp = DataProcessor.initialize([{
"data": "2020-02-24T18:00:00",
"stato": "ITA",
"ricoverati_con_sintomi": 101,
"terapia_intensiva": 26,
"totale_ospedalizzati": 127,
"isolamento_domiciliare": 94,
"totale_positivi": 221,
"variazione_totale_positivi": 0,
"nuovi_positivi": 221,
"dimessi_guariti": 1,
"deceduti": 7,
"totale_casi": 229,
"tamponi": 4324
}, {
"data": "2020-02-25T18:00:00",
"stato": "ITA",
"ricoverati_con_sintomi": 114,
"terapia_intensiva": 35,
"totale_ospedalizzati": 150,
"isolamento_domiciliare": 162,
"totale_positivi": 311,
"variazione_totale_positivi": 90,
"nuovi_positivi": 93,
"dimessi_guariti": 1,
"deceduti": 10,
"totale_casi": 322,
"tamponi": 8623
}, {
"data": "2020-02-26T18:00:00",
"stato": "ITA",
"ricoverati_con_sintomi": 128,
"terapia_intensiva": 36,
"totale_ospedalizzati": 164,
"isolamento_domiciliare": 221,
"totale_positivi": 385,
"variazione_totale_positivi": 74,
"nuovi_positivi": 78,
"dimessi_guariti": 3,
"deceduti": 12,
"totale_casi": 400,
"tamponi": 9587
}], config.DATE_FORMAT)
self.assertEqual(dp.get("total_hospitalized_non_ic"), [101, 114, 128])
self.assertEqual(dp.get("total_intensive_care"), [26, 35, 36])
self.assertEqual(dp.get("total_hospitalized"), [127, 150, 164])
self.assertEqual(dp.get("total_home_confinement"), [94, 162, 221])
self.assertEqual(dp.get("total_active_positives"), [221, 311, 385])
self.assertEqual(dp.get("delta_active_positives"), [0, 90, 74])
self.assertEqual(dp.get("new_infected"), [221, 93, 78])
self.assertEqual(dp.get("total_recovered"), [1, 1, 3])
self.assertEqual(dp.get("total_deaths"), [7, 10, 12])
self.assertEqual(dp.get("total_cases"), [229, 322, 400])
self.assertEqual(dp.get("total_tests"), [4324, 8623, 9587])
def test_get_partial_data(self):
dp = DataProcessor.initialize([{
"data": "2020-02-24T18:00:00",
"stato": "ITA",
"ricoverati_con_sintomi": 101,
"terapia_intensiva": 26,
"totale_ospedalizzati": 127,
"isolamento_domiciliare": 94,
"totale_positivi": 221,
"variazione_totale_positivi": 0,
"nuovi_positivi": 221,
"dimessi_guariti": 1,
"deceduti": 7,
"totale_casi": 229,
"tamponi": 4324
}, {
"data": "2020-02-25T18:00:00",
"stato": "ITA",
"ricoverati_con_sintomi": 114,
"terapia_intensiva": 35,
"totale_ospedalizzati": 150,
"isolamento_domiciliare": 162,
"totale_positivi": 311,
"variazione_totale_positivi": 90,
"nuovi_positivi": 93,
"dimessi_guariti": 1,
"deceduti": 10,
"totale_casi": 322,
"tamponi": 8623
}, {
"data": "2020-02-26T18:00:00",
"stato": "ITA",
"ricoverati_con_sintomi": 128,
"terapia_intensiva": 36,
"totale_ospedalizzati": 164,
"isolamento_domiciliare": 221,
"totale_positivi": 385,
"variazione_totale_positivi": 74,
"nuovi_positivi": 78,
"dimessi_guariti": 3,
"deceduti": 12,
"totale_casi": 400,
"tamponi": 9587
}], config.DATE_FORMAT)
self.assertEqual(dp.get("total_tests", start=1), [8623, 9587])
self.assertEqual(dp.get("total_tests", end=1), [4324])
self.assertEqual(dp.get("total_tests", start=2, end=2), [])
self.assertEqual(dp.get("total_tests", start=1, end=5), [8623, 9587])
def test_create_data_processor_with_one_correct_entry(self):
dp = DataProcessor.initialize([{
"data": "2020-02-28T18:00:00",
"ricoverati_con_sintomi": 345,
"terapia_intensiva": 64,
"totale_ospedalizzati": 409,
"isolamento_domiciliare": 412,
"totale_positivi": 821,
"variazione_totale_positivi": 233,
"nuovi_positivi": 238,
"dimessi_guariti": 46,
"deceduti": 21,
"totale_casi": 888,
"tamponi": 15695,
}], config.DATE_FORMAT)
self.assertEqual(dp.size(), 1)
def test_get_non_existing_property(self):
dp = DataProcessor.initialize([{
"data": "2020-02-28T18:00:00",
"ricoverati_con_sintomi": 345,
"terapia_intensiva": 64,
"totale_ospedalizzati": 409,
"isolamento_domiciliare": 412,
"totale_positivi": 821,
"variazione_totale_positivi": 233,
"nuovi_positivi": 238,
"dimessi_guariti": 46,
"deceduti": 21,
"totale_casi": 888,
"tamponi": 15695,
}], config.DATE_FORMAT)
with self.assertRaises(KeyError):
dp.get("non-existing-key")
def test_data_localization(self):
dp = DataProcessor.initialize([{
"data": "2020-02-26T18:00:00",
"ricoverati_con_sintomi": 345,
"terapia_intensiva": 64,
"totale_ospedalizzati": 409,
"isolamento_domiciliare": 412,
"totale_positivi": 821,
"variazione_totale_positivi": 233,
"nuovi_positivi": 238,
"dimessi_guariti": 46,
"deceduti": 21,
"totale_casi": 888,
"tamponi": 15695,
}], config.DATE_FORMAT)
self.assertIsNone(dp.get("date")[0].tzinfo)
dp.localize_dates("UTC", "Europe/Rome")
self.assertIsNotNone(dp.get("date")[0].tzinfo)
def tweet_updates(dp: DataProcessor, chart_paths):
active_total_cases = dp.get("total_active_positives", start=dp.size() - 2)
active_total_cases_delta = DeltaIndicator(active_total_cases).get_last()
active_total_cases_delta_percentage = DeltaPercentageIndicator(
active_total_cases).get_last()
new_positives = dp.get("new_infected", start=dp.size() - 2)
new_positives_delta = DeltaIndicator(new_positives).get_last()
new_positives_delta_percentage = DeltaPercentageIndicator(
new_positives).get_last()
healed = dp.get("total_recovered", start=dp.size() - 2)
healed_delta = DeltaIndicator(healed).get_last()
healed_delta_percentage = DeltaPercentageIndicator(healed).get_last()
home_confinement = dp.get("total_home_confinement", start=dp.size() - 2)
home_confinement_delta = DeltaIndicator(home_confinement).get_last()
home_confinement_delta_percentage = DeltaPercentageIndicator(
home_confinement).get_last()
total_hospitalized = dp.get("total_hospitalized", start=dp.size() - 2)
total_hospitalized_delta = DeltaIndicator(total_hospitalized).get_last()
total_hospitalized_delta_percentage = DeltaPercentageIndicator(
total_hospitalized).get_last()
total_ic = dp.get("total_intensive_care", start=dp.size() - 2)
total_ic_delta = DeltaIndicator(total_ic).get_last()
total_ic_delta_percentage = DeltaPercentageIndicator(total_ic).get_last()
total_deaths = dp.get("total_deaths", start=dp.size() - 2)
total_deaths_delta = DeltaIndicator(total_deaths).get_last()
total_deaths_delta_percentage = DeltaPercentageIndicator(
total_deaths).get_last()
tests = dp.get("total_tests", start=dp.size() - 2)
tests_delta = DeltaIndicator(tests).get_last()
test_delta_percentage = DeltaPercentageIndicator(tests).get_last()
total_cases = dp.get("total_cases", start=dp.size() - 2)
total_cases_delta = DeltaIndicator(total_cases).get_last()
total_cases_percentage = DeltaPercentageIndicator(total_cases).get_last()
tt = ThreadTwitter(os.getenv("TWITTER_CONSUMER_API_KEY"),
os.getenv("TWITTER_CONSUMER_SECRET_KEY"),
os.getenv("TWITTER_ACCESS_TOKEN_KEY"),
os.getenv("TWITTER_ACCESS_TOKEN_SECRET_KEY"))
tt.set_header("ЪдаЪЄ«ЪЄ╣ Aggiornamento Giornaliero #COVID2019",
repeat=False)
tt.set_footer("Generato da: http://tiny.cc/covid-bot", repeat=False)
data_lines = []
data_lines.append("{0} Casi attivi: {1} ({2:+d}) ({3:+.2f}%)".format(
get_trend_icon(active_total_cases_delta), active_total_cases[1],
active_total_cases_delta, active_total_cases_delta_percentage))
data_lines.append("{0} Nuovi positivi: {1} ({2:+d}) ({3:+.2f}%)".format(
get_trend_icon(new_positives_delta), new_positives[1],
new_positives_delta, new_positives_delta_percentage))
data_lines.append("{0} Guariti/dimessi: {1} ({2:+d}) ({3:+.2f}%)".format(
get_trend_icon(healed_delta), healed[1], healed_delta,
healed_delta_percentage))
data_lines.append(
"{0} Isolamento domiciliare: {1} ({2:+d}) ({3:+.2f}%)".format(
get_trend_icon(home_confinement_delta), home_confinement[1],
home_confinement_delta, home_confinement_delta_percentage))
data_lines.append("{0} Ospedalizzati: {1} ({2:+d}) ({3:+.2f}%)".format(
get_trend_icon(total_hospitalized_delta), total_hospitalized[1],
total_hospitalized_delta, total_hospitalized_delta_percentage))
data_lines.append("{0} Terapie intensive: {1} ({2:+d}) ({3:+.2f}%)".format(
get_trend_icon(total_ic_delta), total_ic[1], total_ic_delta,
total_ic_delta_percentage))
data_lines.append("{0} Morti: {1} ({2:+d}) ({3:+.2f}%)".format(
get_trend_icon(total_deaths_delta), total_deaths[1],
total_deaths_delta, total_deaths_delta_percentage))
data_lines.append("{0} Tamponi: {1} ({2:+d}) ({3:+.2f}%)".format(
get_trend_icon(tests_delta), tests[1], tests_delta,
test_delta_percentage))
data_lines.append("{0} Casi totali: {1} ({2:+d}) ({3:+.2f}%)".format(
get_trend_icon(total_cases_delta), total_cases[1], total_cases_delta,
total_cases_percentage))
for line in data_lines:
tt.add_line(line)
for chart_p in chart_paths:
tt.add_media(chart_p, MediaType.PHOTO)
if not DEBUG_MODE:
tt.tweet()
else:
lines_concat = ""
for line in data_lines:
lines_concat = lines_concat + "\n" + line
log.debug(lines_concat)
def generate_graphs(dp: DataProcessor):
# Prepares data to generate charts
dates = list(map(lambda x: x.date(), dp.get("date")))
positives_active = dp.get("total_active_positives")
deaths = dp.get("total_deaths")
healed = dp.get("total_recovered")
icu = dp.get("total_intensive_care")
non_icu = dp.get("total_hospitalized_non_ic")
home_isolated = dp.get("total_home_confinement")
new_positives = dp.get("new_infected")
tests = DeltaIndicator(dp.get("total_tests")).get_all()
new_healed = DeltaIndicator(dp.get("total_recovered")).get_all()
new_deaths = DeltaIndicator(dp.get("total_deaths")).get_all()
MOVING_AVG_DAYS = 5
new_healed_moving_avg = MovingAverageIndicator(
new_healed, MOVING_AVG_DAYS).get_all()[MOVING_AVG_DAYS - 1:]
new_deaths_moving_avg = MovingAverageIndicator(
new_deaths, MOVING_AVG_DAYS).get_all()[MOVING_AVG_DAYS - 1:]
new_positives_moving_avg = MovingAverageIndicator(
new_positives, MOVING_AVG_DAYS).get_all()[MOVING_AVG_DAYS - 1:]
dates_moving_avg = dates[MOVING_AVG_DAYS - 1:]
# Prepare to make charts
plotly.io.orca.config.default_scale = 2.0
config.TEMP_FILES_PATH.mkdir(parents=True, exist_ok=True)
chart_mgr = ChartManager()
data_time_str = dates[len(dates) - 1].strftime("%d/%m/%Y")
charts_footer = (
"<br>Fonte dati: Protezione Civile Italiana + elaborazioni ({0})"
"<br>Generato da: github.com/berna1995/CovidDailyUpdateBot"
).format(data_time_str)
# Make charts
# Chart 1
graph = go.Figure()
graph.add_trace(
go.Scatter(x=dates,
y=positives_active,
mode="lines+markers",
name="Contagiati Attivi",
line=dict(color=CHART_BLUE)))
graph.add_trace(
go.Scatter(x=dates,
y=deaths,
mode="lines+markers",
name="Deceduti",
line=dict(color=CHART_RED)))
graph.add_trace(
go.Scatter(x=dates,
y=healed,
mode="lines+markers",
name="Guariti",
line=dict(color=CHART_GREEN)))
graph.update_layout(
title="COVID2019 Italia - contagiati attivi, deceduti e guariti",
title_x=0.5,
showlegend=True,
autosize=True,
legend=dict(orientation="h",
xanchor="center",
yanchor="top",
x=0.5,
y=-0.25),
margin=dict(l=30, r=30, t=60, b=150))
graph.update_yaxes(rangemode="normal", automargin=True, ticks="outside")
graph.update_xaxes(tickangle=90,
type="date",
tickformat='%d-%m-%y',
ticks="outside",
tickmode="auto",
nticks=60,
automargin=True)
graph.add_annotation(xref="paper",
yref="paper",
x=0,
yanchor="top",
xanchor="left",
align="left",
y=-0.36,
showarrow=False,
font=dict(size=10),
text=charts_footer)
chart_mgr.add(graph)
# Chart 2
graph = make_subplots(rows=1, cols=3)
graph.add_trace(go.Bar(x=dates,
y=icu,
name="Ospedalizzati TI",
marker=dict(color=CHART_RED)),
row=1,
col=1)
graph.add_trace(go.Bar(x=dates,
y=non_icu,
name="Ospedalizzati Non TI",
marker=dict(color=CHART_BLUE)),
row=1,
col=2)
graph.add_trace(go.Bar(x=dates,
y=home_isolated,
name="Isolamento Domiciliare",
marker=dict(color=CHART_GREEN)),
row=1,
col=3)
graph.update_layout(
title=
"COVID2019 Italia - ospedalizzati e isolamento domiciliare dei positivi",
title_x=0.5,
showlegend=True,
autosize=True,
legend=dict(orientation="h",
xanchor="center",
yanchor="top",
x=0.5,
y=-0.25),
margin=dict(l=30, r=30, t=60, b=150),
bargap=0)
graph.update_yaxes(rangemode="normal", automargin=True, ticks="outside")
graph.update_xaxes(tickangle=90,
type="date",
tickformat='%d-%m-%y',
ticks="outside",
nticks=10,
tickmode="auto",
automargin=True)
graph.add_annotation(xref="paper",
yref="paper",
x=0,
yanchor="top",
xanchor="left",
align="left",
y=-0.36,
showarrow=False,
font=dict(size=10),
text=charts_footer)
chart_mgr.add(graph)
# Chart 3
graph = go.Figure()
graph.add_trace(
go.Bar(x=dates,
y=tests,
name="Tamponi Effettuati",
marker=dict(color=CHART_BLUE)))
graph.add_trace(
go.Bar(x=dates,
y=new_positives,
name="Nuovi Infetti",
marker=dict(color=CHART_RED)))
graph.update_layout(
title=
"COVID2019 Italia - tamponi effettuati giornalmente e nuovi infetti",
title_x=0.5,
showlegend=True,
autosize=True,
legend=dict(orientation="h",
xanchor="center",
yanchor="top",
x=0.5,
y=-0.25),
margin=dict(l=30, r=30, t=60, b=150),
barmode="group",
bargap=0)
graph.update_yaxes(rangemode="normal", automargin=True, ticks="outside")
graph.update_xaxes(tickangle=90,
type="date",
tickformat='%d-%m-%y',
ticks="outside",
rangemode="normal",
tickmode="auto",
nticks=60,
automargin=True)
graph.add_annotation(xref="paper",
yref="paper",
x=0,
yanchor="top",
xanchor="left",
align="left",
y=-0.36,
showarrow=False,
font=dict(size=10),
text=charts_footer)
chart_mgr.add(graph)
# Chart 4
graph = go.Figure()
graph.add_trace(
go.Scatter(x=dates_moving_avg,
y=new_positives_moving_avg,
mode="lines+markers",
name="Infetti",
line=dict(color=CHART_BLUE)))
graph.add_trace(
go.Scatter(x=dates_moving_avg,
y=new_healed_moving_avg,
mode="lines+markers",
name="Guariti",
line=dict(color=CHART_GREEN)))
graph.add_trace(
go.Scatter(x=dates_moving_avg,
y=new_deaths_moving_avg,
mode="lines+markers",
name="Morti",
line=dict(color=CHART_RED)))
graph.update_layout(
title=
"COVID2019 Italia - nuovi guariti, morti, infetti [media mobile {0}gg]"
.format(MOVING_AVG_DAYS),
title_x=0.5,
showlegend=True,
autosize=True,
legend=dict(orientation="h",
xanchor="center",
yanchor="top",
x=0.5,
y=-0.25),
margin=dict(l=30, r=30, t=60, b=150))
graph.update_yaxes(rangemode="normal", automargin=True, ticks="outside")
graph.update_xaxes(tickangle=90,
type="date",
tickformat='%d-%m-%y',
ticks="outside",
tickmode="auto",
nticks=60,
automargin=True)
graph.add_annotation(xref="paper",
yref="paper",
x=0,
yanchor="top",
xanchor="left",
align="left",
y=-0.36,
showarrow=False,
font=dict(size=10),
text=charts_footer)
chart_mgr.add(graph)
gen_paths = chart_mgr.generate_images(config.TEMP_FILES_PATH)
plotly.io.orca.shutdown_server()
return gen_paths
def test_create_data_processor_with_invalid_data_type(self):
with self.assertRaises(InvalidDataFormatException):
DataProcessor.initialize(12345, config.DATE_FORMAT)
def test_create_data_process_from_json_empty_list(self):
try:
dp = DataProcessor.initialize("[]", config.DATE_FORMAT)
self.assertEqual(dp.size(), 0)
except Exception:
self.fail("should not fail, empty JSON data should be accepted")