def _get_points(self,
data: typing.List[typing.Dict],
min_timestamp=None) -> typing.List[CovidPoint]:
res = []
for entry in data:
updated_at = datetime.datetime.strptime(entry['dateChecked'],
'%Y-%m-%dT%H:%M:%SZ')
if min_timestamp and updated_at < datetime.datetime.fromtimestamp(
min_timestamp):
continue
point = CovidPoint(
source="covidtracking_usa",
timestamp=updated_at,
last_update=int(updated_at.strftime('%s')),
country="USA",
region=entry['state'].replace("'", "."),
city="",
confirmed_cumulative=int(entry['total'] or 0),
death_cumulative=int(entry['death'] or 0),
recovered_cumulative=entry['negative'] or 0,
hospitalized_cumulative=int(entry['hospitalized'] or 0),
severe_cumulative=0,
tests_cumulative=int(entry['totalTestResults'] or 0))
res.append(point)
return res
def _get_points(self,
data: typing.List[typing.List],
min_timestamp=None) -> typing.List[CovidPoint]:
res = []
for entry in data:
updated_at, rows = entry
rows = self._parse_csv(rows)
for row in rows:
if min_timestamp and updated_at < datetime.datetime.fromtimestamp(
min_timestamp):
continue
if 'undesland' in row[0] or 'tate' in row[0] or 'esamt' in row[
0]:
continue
point = CovidPoint(
source="rki_de",
timestamp=updated_at,
last_update=int(updated_at.strftime('%s')),
country="Germany",
region=row[0],
city="",
confirmed_cumulative=int(row[1].replace('.', '').strip()
or 0),
death_cumulative=int(row[2].replace('.', '').strip(
) if len(row) > 2 and row[2] else 0),
recovered_cumulative=0,
hospitalized_cumulative=0,
severe_cumulative=0,
tests_cumulative=0)
res.append(point)
return res
def get_nearest_historical_points_by_fields(self, timestamp: int, source: str, field: str) \
-> typing.List[CovidPoint]:
assert isinstance(timestamp, int)
query = "select *, country, region, city " \
"from {} " \
"where time < {} " \
"group by {} order by desc limit 1".format(
source, timestamp * 10**9, field
)
points = self.influxdb_client.query(query, epoch='s', database='covid19').get_points()
res = [
CovidPoint(
source=source,
timestamp=point['time'],
last_update=point['time'],
country=point['country'],
region=point['region'],
city=point['city'],
confirmed_cumulative=point['confirmed_cumulative'],
hospitalized_cumulative=point['hospitalized_cumulative'],
severe_cumulative=point['severe_cumulative'],
death_cumulative=point['death_cumulative'],
recovered_cumulative=point['recovered_cumulative'],
tests_cumulative=point.get('tests_cumulative', 0)
) for point in points
]
return res
def _get_points(self,
data: str,
min_timestamp=None) -> typing.List[CovidPoint]:
res = []
rows = self._parse_csv(data)
for row in rows:
if 'data' == row[0]:
continue
dt_format = '%Y-%m-%dT%H:%M:%S' if 'T' in row[
0] else '%Y-%m-%d %H:%M:%S'
updated_at = datetime.datetime.strptime(row[0], dt_format)
if min_timestamp and updated_at < datetime.datetime.fromtimestamp(
min_timestamp):
continue
point = CovidPoint(source="dpc_ita",
timestamp=updated_at,
last_update=int(updated_at.strftime('%s')),
country="Italy",
region=row[3].replace("'", "."),
city="",
confirmed_cumulative=int(row[14]),
death_cumulative=int(row[13]),
recovered_cumulative=int(row[12]),
hospitalized_cumulative=int(row[8]),
severe_cumulative=int(row[7]),
tests_cumulative=int(row[15]))
res.append(point)
return res
def _get_points(self,
data: typing.List[typing.List],
min_timestamp=None) -> typing.List[CovidPoint]:
res = []
for entry in data:
updated_at, rows = entry
rows = self._parse_csv(rows)
error = False
for v in range(1, 8):
try:
parser = getattr(self, '_get_points_from_csv_v' + str(v))
rows = parser(rows)
error = False
break
except (IndexError, AssertionError):
error = True
if error:
raise ValueError('Error parsing row: %s' % rows)
for row in rows:
if min_timestamp and updated_at < datetime.datetime.fromtimestamp(
min_timestamp):
continue
if 'ases' in row[1]:
continue
point = CovidPoint(
source="worldometers",
timestamp=updated_at,
last_update=int(updated_at.strftime('%s')),
country=row[0],
region="",
city="",
confirmed_cumulative=int(row[1].replace(',', '').strip()
or 0),
death_cumulative=int(row[2].replace(',', '').strip() or 0),
recovered_cumulative=int(row[3].replace(',', '').strip()
or 0),
hospitalized_cumulative=0,
severe_cumulative=int(row[4].replace(',', '').strip()
or 0),
tests_cumulative=int(row[5].replace(',', '').strip() or 0),
)
res.append(point)
return res
def _get_points(self,
data: typing.List[typing.List]) -> typing.List[CovidPoint]:
res = []
for entry in data:
rows = self._parse_csv(entry[1])
for row in rows:
last_update = self._get_last_update_timestamp(row[0])
point = CovidPoint(
source="csse",
timestamp=datetime.datetime.fromtimestamp(last_update),
last_update=last_update,
country=row[1],
region=row[2],
city="",
confirmed_cumulative=int(row[3] or 0),
death_cumulative=int(row[4] or 0),
recovered_cumulative=int(row[5] or 0),
hospitalized_cumulative=0,
severe_cumulative=0,
tests_cumulative=0)
res.append(point)
return res
def _get_points(self, data: typing.List[typing.Dict], min_timestamp=None) -> typing.List[CovidPoint]:
res = []
for row in data:
dt_format = '%Y-%m-%dT%H:%M:%S' if 'T' in row['data'] else '%Y-%m-%d %H:%M:%S'
updated_at = datetime.datetime.strptime(row['data'], dt_format)
if min_timestamp and updated_at < datetime.datetime.fromtimestamp(min_timestamp):
continue
point = CovidPoint(
source="dpc_ita",
timestamp=updated_at,
last_update=int(updated_at.strftime('%s')),
country="Italy",
region=normalize_data(row['denominazione_regione'].replace("'", ".")),
city="",
confirmed_cumulative=int(row['totale_positivi']),
death_cumulative=int(row['deceduti']),
recovered_cumulative=int(row['dimessi_guariti']),
hospitalized_cumulative=int(row['totale_ospedalizzati']),
severe_cumulative=int(row['terapia_intensiva']),
tests_cumulative=int(row['tamponi'])
)
res.append(point)
return res