def __init__(self):
URLBase.__init__(self,
output_dir=get_overseas_dir() / 'pt' / 'data',
urls_dict={
'national_data.json':
URL(NATIONAL_URL.replace(' ', '%20'),
static_file=False),
'municipality_data.json':
URL(MUNICIPALITY_URL.replace(' ', '%20'),
static_file=False)
})
self.sdpf = StrictDataPointsFactory(region_mappings={
('pt_municipality', 'pt', 'guimarães'):
None,
('pt_municipality', 'pt', 'tavira'):
None,
('pt_municipality', 'pt', 'lagoa (faro)'):
None,
('pt_municipality', 'pt', 'vila da praia da vitória'):
None,
('pt_municipality', 'pt', 'ponte de sor'):
None,
('pt_municipality', 'pt', 'calheta (açores)'):
None,
('pt_municipality', 'pt', 'calheta [r.a. açores]'):
None,
('pt_municipality', 'pt', 'lagoa [r.a. açores]'):
None,
('pt_municipality', 'pt', 'calheta de são jorge'):
None,
('pt_municipality', 'pt', 'calheta [r.a. madeira]'):
None,
},
mode=MODE_STRICT)
self.update()
def __init__(self):
# Only raw_data4.json is currently being updated,
# so won't download the others every day
URLBase.__init__(self,
output_dir=get_overseas_dir() / 'id' / 'data',
urls_dict={
'provinces_1.csv': URL(
'https://docs.google.com/spreadsheet/ccc?'
'key=1ma1T9hWbec1pXlwZ89WakRk-OfVUQZsOCFl4FwZxzVw&gid=2052139453&output=csv',
static_file=False
),
'provinces_2.csv': URL(
'https://docs.google.com/spreadsheet/ccc?'
'key=1sgiz8x71QyIVJZQguYtG9n6xBEKdM4fXuDs_d8zKOmY&gid=83750310&output=csv',
static_file=False
)
}
)
self.sdpf = StrictDataPointsFactory(
region_mappings={
('admin_1', 'id', 'babel'): ('admin_1', 'id', 'ID-BB'),
('admin_1', 'id', 'diy'): ('admin_1', 'id', 'ID-YO'),
('admin_1', 'id', 'kaltara'): None, #('admin_1', 'id', 'ID-KU'),
('admin_1', 'id', 'kep riau'): ('admin_1', 'id', 'ID-KR'),
('admin_1', 'id', 'papbar'): ('admin_1', 'id', 'ID-PB'),
('admin_1', 'id', 'sulbar'): ('admin_1', 'id', 'ID-SR'),
},
mode=MODE_STRICT
)
self.update()
def __init__(self):
URLBase.__init__(self,
output_dir=get_overseas_dir() / 'world_covid19datahub' / 'data',
urls_dict={
'rawdata-1.csv': URL(url='https://storage.covid19datahub.io/rawdata-1.csv', static_file=False),
'rawdata-2.csv': URL(url='https://storage.covid19datahub.io/rawdata-2.csv', static_file=False),
'rawdata-3.csv': URL(url='https://storage.covid19datahub.io/rawdata-3.csv', static_file=False),
}
)
self.update()
self.sdpf = StrictDataPointsFactory(mode=MODE_DEV) # NOTE ME!!!
def __init__(self):
URLBase.__init__(self,
output_dir=get_overseas_dir() / 'bw' / 'data',
urls_dict={
'data_1.json': URL('https://services7.arcgis.com/mbEujVpJG0aE29fS/arcgis/rest/services/Covid19_View/FeatureServer/0/query?f=json&where=1%3D1&returnGeometry=true&spatialRel=esriSpatialRelIntersects&maxAllowableOffset=1222&geometry=%7B%22xmin%22%3A2504688.542842917%2C%22ymin%22%3A-3130860.678554911%2C%22xmax%22%3A3130860.678554911%2C%22ymax%22%3A-2504688.542842917%2C%22spatialReference%22%3A%7B%22wkid%22%3A102100%7D%7D&geometryType=esriGeometryEnvelope&inSR=102100&outFields=*&outSR=102100&resultType=tile',
static_file=False),
'data_2.json': URL('https://services7.arcgis.com/mbEujVpJG0aE29fS/arcgis/rest/services/Covid19_View/FeatureServer/0/query?f=json&where=1%3D1&returnGeometry=true&spatialRel=esriSpatialRelIntersects&maxAllowableOffset=1222&geometry=%7B%22xmin%22%3A2504688.542842917%2C%22ymin%22%3A-2504688.542842917%2C%22xmax%22%3A3130860.678554911%2C%22ymax%22%3A-1878516.4071309231%2C%22spatialReference%22%3A%7B%22wkid%22%3A102100%7D%7D&geometryType=esriGeometryEnvelope&inSR=102100&outFields=*&outSR=102100&resultType=tile',
static_file=False)
}
)
self.sdpf = StrictDataPointsFactory(mode=MODE_STRICT)
self.update()
def __init__(self):
# Only raw_data4.json is currently being updated,
# so won't download the others every day
URLBase.__init__(
self,
output_dir=get_data_dir() / 'vic' / 'tableau_native',
urls_dict={
'agegroup.json':
URL('https://public.tableau.com/workbooks/Agegroup_15982346382420.twb',
static_file=False),
'genderagegroup.json':
URL('https://public.tableau.com/workbooks/GenderAgeGroup.twb',
static_file=False),
'transmissions.json':
URL('https://public.tableau.com/workbooks/Transmissions.twb',
static_file=False),
'transmissions_over_time.json':
URL('https://public.tableau.com/workbooks/Transmissionsovertime.twb',
static_file=False),
'active_cases.json':
URL('https://public.tableau.com/workbooks/Activecases_15982341517530.twb',
static_file=False),
'cases.json':
URL('https://public.tableau.com/workbooks/Cases_15982342702770.twb',
static_file=False),
# TODO: Support healthcare workers graphs!
'hcw_source_infections.json':
URL('https://public.tableau.com/workbooks/HCWSourceInfections.twb',
static_file=False),
'hcw_cases.json':
URL('https://public.tableau.com/workbooks/HCWCases.twb',
static_file=False),
})
self.update()
def __init__(self):
URLBase.__init__(
self,
output_dir=get_overseas_dir() / 'tr' / 'data',
urls_dict={
'regions.json':
URL('https://services5.arcgis.com/26ObnytmBZ58Wbj8/arcgis/rest/services/Kovid_19_TurkiyeIL/FeatureServer/0/query?f=json&where=1%3D1&returnGeometry=false&spatialRel=esriSpatialRelIntersects&outFields=*&orderByFields=T_KesinVaka%20desc&resultOffset=0&resultRecordCount=81&resultType=standard&cacheHint=true',
static_file=False),
'other_stats.json':
URL('https://services1.arcgis.com/5hM12kUbM5soE48F/arcgis/rest/services/Case_time_v4/FeatureServer/0/query?f=json&where=Turkey_Date%20BETWEEN%20timestamp%20%272019-12-31%2013%3A00%3A00%27%20AND%20CURRENT_TIMESTAMP&returnGeometry=false&spatialRel=esriSpatialRelIntersects&outFields=*&orderByFields=Turkey_Date%20asc&resultOffset=0&resultRecordCount=32000&resultType=standard',
static_file=False)
})
self.sdpf = StrictDataPointsFactory(mode=MODE_STRICT)
self.update()
def __get_urls_dict(self):
r = {}
def daterange(start_date, end_date):
for n in range(int((end_date - start_date).days)):
yield start_date + timedelta(n)
start_date = date.today() - timedelta(days=14)
#start_date = date(2020, 3, 18)
end_date = date.today()
for single_date in daterange(start_date, end_date):
url = (
'https://services1.arcgis.com/5PzxEwuu4GtMhqQ6/arcgis/rest/services/Synthese_Covid19_France/FeatureServer/0/query?'
'f=json&'
f'where=(OBJECTID%20%3E%200)%20AND%20(Jour=%27{single_date.strftime("%d/%m/%Y")}%27)&'
'returnGeometry=false&'
'spatialRel=esriSpatialRelIntersects&'
'maxAllowableOffset=2445&'
'geometryType=esriGeometryEnvelope&inSR=102100&'
'outFields=*&outSR=102100&'
'resultType=tile'
)
r[f'data_{single_date.strftime("%Y_%m_%d")}.json'] = URL(url, False)
return r
def __init__(self):
URLBase.__init__(
self,
output_dir=get_overseas_dir() / 'si' / 'data',
urls_dict={
# Points for each county w/stats
'municipalities.json':
URL('https://api.sledilnik.org/api/municipalities',
static_file=False),
})
self.sdpf = StrictDataPointsFactory(region_mappings={
('admin_1', 'si', 'si-010'): None,
('admin_1', 'si', 'si-142'): None,
('admin_1', 'si', 'si-207'): None,
('admin_1', 'si', 'si-199'): None,
('admin_1', 'si', 'si-206'): None,
('admin_1', 'si', 'si-212'): None,
('admin_1', 'si', 'si-211'): None,
('admin_1', 'si', 'si-208'): None,
('admin_1', 'si', 'si-201'): None,
('admin_1', 'si', 'si-196'): None,
('admin_1', 'si', 'si-197'): None,
('admin_1', 'si', 'si-209'): None,
('admin_1', 'si', 'si-204'): None,
('admin_1', 'si', 'si-213'): None,
('admin_1', 'si', 'si-200'): None,
('admin_1', 'si', 'si-205'): None,
('admin_1', 'si', 'si-203'): None,
('admin_1', 'si', 'si-198'): None,
('admin_1', 'si', 'si-202'): None,
('admin_1', 'si', 'si-088'): None,
('admin_1', 'si', 'si-210'): None,
},
mode=MODE_STRICT)
self.update()
def __init__(self):
URLBase.__init__(
self,
output_dir=get_overseas_dir() / 'ie' / 'data',
urls_dict={
# Points for each county w/stats
'county_data.csv':
URL('http://opendata-geohive.hub.arcgis.com/datasets/4779c505c43c40da9101ce53f34bb923_0.csv?outSR={%22latestWkid%22:3857,%22wkid%22:102100}',
static_file=False),
# Country-wide stats (age distribution etc)
'country_data.csv':
URL('http://opendata-geohive.hub.arcgis.com/datasets/d8eb52d56273413b84b0187a4e9117be_0.csv?outSR={%22latestWkid%22:3857,%22wkid%22:102100}',
static_file=False)
})
self.sdpf = StrictDataPointsFactory(mode=MODE_STRICT)
self.update()
def __init__(self):
# Only raw_data4.json is currently being updated,
# so won't download the others every day
URLBase.__init__(
self,
output_dir=get_overseas_dir() / 'so' / 'data',
urls_dict={
'so_data.csv':
URL('https://docs.google.com/spreadsheets/d/e/2PACX-1vRTGuZDNylQKqZC7ITpHkLw-7nHvElQNtImJS7kRFXGak664t6jxDjvdVHHWkKPJ7rvwAtj6VGXrQUC/pub?output=csv',
static_file=False)
})
self.sdpf = StrictDataPointsFactory(
region_mappings={
('admin_1', 'so', 'banadir'): ('admin_1', 'so', 'so-bn'),
('admin_1', 'so', 'galmudug state'):
('admin_1', 'so', 'so-ga'),
('admin_1', 'so', 'hir-shabelle state'): None, # FIXME!
('admin_1', 'so', 'jubaland state'): None, # FIXME!
('admin_1', 'so', 'puntland'): None, # FIXME!!
('admin_1', 'so', 'somaliland'): None, # FIXME!
('admin_1', 'so', 'south west state'): None, # FIXME!
('admin_1', 'so', 'south west'): None, # FIXME!
('admin_1', 'so', 'jubaland'): None,
('admin_1', 'so', 'galmmudug'): ('admin_1', 'so', 'so-ga'),
('admin_1', 'so', 'hir-shabelle'): None,
('admin_1', 'so', 'galmudug'): ('admin_1', 'so', 'so-ga'),
},
mode=MODE_STRICT)
self.update()
def __init__(self):
# Only raw_data4.json is currently being updated,
# so won't download the others every day
URLBase.__init__(self,
output_dir=get_overseas_dir() / 'et' / 'data',
urls_dict={'gh_data.csv': URL('', static_file=False)})
self.update()
def __init__(self):
# Only raw_data4.json is currently being updated,
# so won't download the others every day
URLBase.__init__(
self,
output_dir=get_overseas_dir() / 'af' / 'data',
urls_dict={
'provinces_1.csv':
URL(
'https://docs.google.com/spreadsheet/ccc?'
'key=1F-AMEDtqK78EA6LYME2oOsWQsgJi4CT3V_G4Uo-47Rg&'
'gid=1539509351&output=csv',
static_file=False)
})
self.sdpf = StrictDataPointsFactory(
region_mappings={
('admin_1', 'af', 'dykundi'):
None, #('admin_1', 'af', 'AF-DAY'),
('admin_1', 'af', 'hirat'): ('admin_1', 'af', 'AF-HER'),
('admin_1', 'af', 'jawzjan'): ('admin_1', 'af', 'AF-JOW'),
('admin_1', 'af', 'paktya'): ('admin_1', 'af', 'AF-PIA'),
('admin_1', 'af', 'panjsher'):
None, #('admin_1', 'af', 'AF-PAN'),
('admin_1', 'af', 'sar-e-pul'): ('admin_1', 'af', 'AF-SAR'),
('admin_1', 'af', 'daykundi'):
None, #('admin_1', 'af', 'AF-DAY'),
('admin_1', 'af', 'panjshir'):
None, #('admin_1', 'af', 'AF-PAN'),
('admin_1', 'af', 'panjshir\xa0province'):
None, #('admin_1', 'af', 'AF-PAN'),
('admin_1', 'af', 'sar-e\xa0pol\xa0province'):
('admin_1', 'af', 'AF-SAR'),
},
mode=MODE_STRICT)
self.update()
def __init__(self):
URLBase.__init__(self,
output_dir=get_overseas_dir() / 'lt' / 'data',
urls_dict={
'regions_data.json': URL(DATA_URL,
static_file=False),
})
self.sdpf = StrictDataPointsFactory(mode=MODE_STRICT)
def __init__(self):
# Only raw_data4.json is currently being updated,
# so won't download the others every day
URLBase.__init__(self,
output_dir=get_overseas_dir() / '' / '',
urls_dict={'': URL('', static_file=False)})
if not WestAfricaPowerBI.data_downloaded_today():
wapb = WestAfricaPowerBI()
wapb.run_powerbi_grabber()
def __init__(self):
# Only raw_data4.json is currently being updated,
# so won't download the others every day
URLBase.__init__(self,
output_dir=get_overseas_dir() / 'gh' / 'data2',
urls_dict={
'regions.json': URL(regions_url, static_file=False),
'recoveries.json': URL(recoveries_url, static_file=False),
'deaths.json': URL(deaths_url, static_file=False),
'confirmed.json': URL(confirmed_url, static_file=False)
}
)
self.sdpf = StrictDataPointsFactory(
region_mappings={
},
mode=MODE_STRICT
)
self.update()
def __init__(self):
URLBase.__init__(self,
output_dir=get_overseas_dir() / 'is' / 'data',
urls_dict={
'is_index.html': URL('https://e.infogram.com/e3205e42-19b3-4e3a-a452-84192884450d?src=embed',
static_file=False)
}
)
self.update()
def __init__(self):
# Only raw_data4.json is currently being updated,
# so won't download the others every day
URLBase.__init__(
self,
output_dir=get_overseas_dir() / 've' / 'data',
urls_dict={
'cases_deaths.csv':
URL(
#'https://docs.google.com/spreadsheets/d/e/2PACX-1vTT8ef-uVa5q_5kBYbVXeEpRCW8gOJHlWhHGrH8dQ704D64_yNaMMjvkzdgD9YweSBQ-GyqnGLLasvK/pub?gid=608241994&single=true&output=csv',
'https://docs.google.com/spreadsheets/d/e/2PACX-1vTT8ef-uVa5q_5kBYbVXeEpRCW8gOJHlWhHGrH8dQ704D64_yNaMMjvkzdgD9YweSBQ-GyqnGLLasvK/pub?gid=754668318&single=true&output=csv',
static_file=False),
'cases_by_age_gender.csv':
URL('https://docs.google.com/spreadsheets/d/e/2PACX-1vTT8ef-uVa5q_5kBYbVXeEpRCW8gOJHlWhHGrH8dQ704D64_yNaMMjvkzdgD9YweSBQ-GyqnGLLasvK/pub?gid=1574343536&single=true&output=csv',
static_file=False)
})
self.sdpf = StrictDataPointsFactory(mode=MODE_STRICT)
self.update()
def __init__(self):
URLBase.__init__(self,
output_dir=get_overseas_dir() / 'tr' / 'wikidata',
urls_dict={
'covid19_in_turkey.html': URL(WIKI_URL, static_file=False)
}
)
self.sdpf = StrictDataPointsFactory(mode=MODE_STRICT)
self.update()
def __init__(self):
URLBase.__init__(self,
output_dir=get_overseas_dir() / 'ps' / 'data',
urls_dict={
'ps_corona.html':
URL('https://www.corona.ps/details',
static_file=False)
})
self.sdpf = StrictDataPointsFactory(mode=MODE_STRICT)
self.update()
def __init__(self):
URLBase.__init__(self,
output_dir=get_overseas_dir() / 'mk' / 'data',
urls_dict={
'mk_corona.html':
URL('https://koronavirus.gov.mk/vesti/218055',
static_file=False)
})
self.sdpf = StrictDataPointsFactory(mode=MODE_STRICT)
self.update()
def __init__(self):
URLBase.__init__(self,
output_dir=get_overseas_dir() / 'es' / 'isciii_data',
urls_dict={
'index.html':
URL('https://cnecovid.isciii.es/covid19/',
static_file=False),
})
self.sdpf = StrictDataPointsFactory(mode=MODE_STRICT)
self.update()
def __init__(self):
URLBase.__init__(self,
output_dir=get_overseas_dir() / 'ma' / 'data',
urls_dict={
'index.html':
URL('https://covid.hespress.com/',
static_file=False)
})
self.sdpf = StrictDataPointsFactory(mode=MODE_STRICT)
self.update()
def __init__(self):
URLBase.__init__(
self,
output_dir=get_overseas_dir() / 'mw' / 'data',
urls_dict={
'aggregates.json':
URL('https://covid19.health.gov.mw:3000/api/v0/aggregates',
static_file=False),
'district_aggregates.json':
URL('https://covid19.health.gov.mw:3000/api/v0/districts/aggregates',
static_file=False)
})
self.sdpf = StrictDataPointsFactory(region_mappings={
('admin_1', 'mw', 'mw-kr'):
None,
('admin_1', 'mw', ''): ('MERGE', 'admin_1', 'mw', 'unknown'),
},
mode=MODE_STRICT)
self.update()
def __init__(self):
URLBase.__init__(
self,
output_dir=get_overseas_dir() / 'th' / 'data',
urls_dict={
'covid_19_daily.xlsx':
URL('https://data.go.th/dataset/8a956917-436d-4afd-a2d4-59e4dd8e906e/resource/24ac8406-0cf9-4f8e-a55e-b53cf6766d1a/download/pm-covid19-adj.xlsx',
static_file=False),
'cases.json':
URL('https://covid19.th-stat.com/api/open/cases',
static_file=False),
'timeline.json':
URL('https://covid19.th-stat.com/api/open/timeline',
static_file=False),
'sum.json':
URL('https://covid19.th-stat.com/api/open/cases/sum',
static_file=False)
})
self.districts_map = get_districts_map()
self.update()
def __init__(self):
URLBase.__init__(
self,
output_dir=get_overseas_dir() / 'world_who' / 'data',
urls_dict={
'WHO-COVID-19-global-data.csv':
URL(url='https://covid19.who.int/WHO-COVID-19-global-data.csv',
static_file=False),
})
self.update()
self.sdpf = StrictDataPointsFactory(mode=MODE_STRICT)
def __init__(self):
URLBase.__init__(
self,
output_dir=get_overseas_dir() / 'gr' / 'data',
urls_dict={
'regions.json':
URL('https://covid-19-greece.herokuapp.com/regions-history',
static_file=False),
})
self.sdpf = StrictDataPointsFactory(mode=MODE_STRICT)
self.update()
def __init__(self):
URLBase.__init__(self,
output_dir=get_overseas_dir() / 'ye' / 'data',
urls_dict={
'totalGovDataForTable.json': URL(
'https://yemen-corona.com/index.php?route=report/statistics/totalGovDataForTable',
static_file=False
)
}
)
self.sdpf = StrictDataPointsFactory(mode=MODE_STRICT)
self.update()
def __get_urls_dict(self):
r = {}
for url, key in (
('https://opendata.ecdc.europa.eu/covid19/nationalcasedeath_eueea_daily_ei/xlsx/data.xlsx',
'world_data.xlsx'),
('https://opendata.ecdc.europa.eu/covid19/hospitalicuadmissionrates/xlsx/data.xlsx',
'hosp_icu.xlsx'),
('https://opendata.ecdc.europa.eu/covid19/testing/xlsx/data.xlsx',
'tests.xlsx'),
):
r[key] = URL(url, static_file=False)
return r
def __init__(self):
URLBase.__init__(
self,
output_dir=get_overseas_dir() / 'cn' / 'qqdata',
urls_dict={
'data.json':
URL('https://view.inews.qq.com/g2/getOnsInfo?name=disease_h5',
static_file=False)
},
)
self.sdpf = StrictDataPointsFactory(region_mappings={}, mode=MODE_DEV)
self.update()
def __init__(self):
URLBase.__init__(
self,
output_dir=get_data_dir() / 'vic' / 'csv_data',
urls_dict={
'lga.json':
URL('https://docs.google.com/spreadsheets/d/e/2PACX-1vQ9oKYNQhJ6v85dQ9qsybfMfc-eaJ9oKVDZKx-VGUr6szNoTbvsLTzpEaJ3oW_LZTklZbz70hDBUt-d/pub?gid=0&single=true&output=csv',
static_file=False),
'postcode.json':
URL('https://docs.google.com/spreadsheets/d/e/2PACX-1vTwXSqlP56q78lZKxc092o6UuIyi7VqOIQj6RM4QmlVPgtJZfbgzv0a3X7wQQkhNu8MFolhVwMy4VnF/pub?gid=0&single=true&output=csv',
static_file=False),
'agegroup.csv':
URL('https://www.dhhs.vic.gov.au/ncov-covid-cases-by-age-group-csv',
static_file=False),
'all_lga.csv':
URL('https://www.dhhs.vic.gov.au/ncov-covid-cases-by-lga-csv',
static_file=False),
'all_lga_acquired_source':
URL('https://www.dhhs.vic.gov.au/ncov-covid-cases-by-lga-source-csv',
static_file=False),
'all_acquired_source':
URL('https://www.dhhs.vic.gov.au/ncov-covid-cases-by-source-csv',
static_file=False)
})
self.update()
