def __init__(self):
URLBase.__init__(self,
output_dir=get_overseas_dir() / 'lv' / 'data',
urls_dict={
'regions_data.json':
URL(REGIONS_URL, static_file=False),
})
self.sdpf = StrictDataPointsFactory(region_mappings={
('admin_1', 'lv', 'daugavpils novads'):
None,
('admin_1', 'lv', 'jelgavas novads'):
None,
('admin_1', 'lv', 'jēkabpils novads'):
None,
('admin_1', 'lv', 'kocēnu novads'):
None,
('admin_1', 'lv', 'līgatnes novads'):
None,
('admin_1', 'lv', 'pārgaujas novads'):
None,
('admin_1', 'lv', 'pārgaujas novads'):
None,
('admin_1', 'lv', 'priekuļu novads'):
None,
('admin_1', 'lv', 'rēzeknes novads'):
None,
('admin_1', 'lv', 'rūjienas novads'):
None,
('admin_1', 'lv', 'strenču novads'):
None,
('admin_1', 'lv', 'varakļānu novads'):
None,
('admin_1', 'lv', 'ventspils novads'):
None,
('admin_1', 'lv', 'viļānu novads'):
None,
},
mode=MODE_STRICT)
def __init__(self):
URLBase.__init__(
self,
output_dir=get_overseas_dir() / 'sa' / 'data',
urls_dict={
'recovered_sum.json':
URL('https://services6.arcgis.com/bKYAIlQgwHslVRaK/arcgis/rest/services/VWPlacesUniqueWithStatistics/FeatureServer/1/query?f=json&where=Recovered_SUM%3E0&returnGeometry=false&spatialRel=esriSpatialRelIntersects&outFields=*&orderByFields=Recovered_SUM%20desc&outSR=102100&resultOffset=0&resultRecordCount=1000&resultType=standard&cacheHint=true',
static_file=False),
'tested_sum.json':
URL('https://services6.arcgis.com/bKYAIlQgwHslVRaK/arcgis/rest/services/VWPlacesUniqueWithStatistics01/FeatureServer/1/query?f=json&where=Tested_SUM%3E0&returnGeometry=false&spatialRel=esriSpatialRelIntersects&outFields=*&orderByFields=Tested_SUM%20desc&outSR=102100&resultOffset=0&resultRecordCount=1000&resultType=standard&cacheHint=true',
static_file=False),
'confirmed_sum.json':
URL('https://services6.arcgis.com/bKYAIlQgwHslVRaK/arcgis/rest/services/VWPlacesUniqueWithStatistics/FeatureServer/1/query?f=json&where=1%3D1&returnGeometry=false&spatialRel=esriSpatialRelIntersects&outFields=*&orderByFields=Confirmed_SUM%20desc&outSR=102100&resultOffset=0&resultRecordCount=1000&resultType=standard&cacheHint=true',
static_file=False)
})
self.sdpf = StrictDataPointsFactory(region_mappings={
('admin_1', 'sa', 'eastern region'): ('admin_1', 'sa', 'SA-04'),
('admin_1', 'sa', 'al qaseem'): ('admin_1', 'sa', 'SA-05'),
('admin_1', 'sa', 'aseer'): ('admin_1', 'sa', 'SA-14'),
('admin_1', 'sa', 'northern borders'): ('admin_1', 'sa', 'SA-08'),
},
mode=MODE_STRICT)
self.update()
def __init__(self):
# Only raw_data4.json is currently being updated,
# so won't download the others every day
URLBase.__init__(
self,
output_dir=get_overseas_dir() / 'et' / 'data',
urls_dict={
'et_data.csv':
URL(
'https://docs.google.com/spreadsheets/d/'
'1wpMiswWu5En3Ljoqv-N9gK_VQWU6xYA9NBAYzJ3aE2g/'
'export?format=csv&'
'id=1wpMiswWu5En3Ljoqv-N9gK_VQWU6xYA9NBAYzJ3aE2g',
static_file=False)
})
self.sdpf = StrictDataPointsFactory(region_mappings={
('admin_1', 'et', 'snnpr'): ('admin_1', 'et', 'et-sn'),
('admin_1', 'et', 'sidama'):
None,
('admin_1', 'et', 'benshangul gumuz'): ('admin_1', 'et', 'et-be'),
},
mode=MODE_STRICT)
self.update()
def __init__(self):
URLBase.__init__(self,
output_dir=get_overseas_dir() / 'cz' / 'data',
urls_dict={
'national_tests.json':
URL(TOTAL_TESTS_URL, static_file=False),
'national_confirmed.json':
URL(TOTAL_CONFIRMED, static_file=False),
'national_stats.json':
URL(TOTAL_NATIONAL_STATS, static_file=False),
'regional_cases.json':
URL(INDIVIDUAL_CASES, static_file=False),
'regional_totals.json':
URL(REGIONAL_TOTALS, static_file=False),
'regional_recovered.json':
URL(RECOVERED, static_file=False),
'regional_deaths.json':
URL(DEATHS, static_file=False),
})
self.sdpf = StrictDataPointsFactory(region_mappings={
('cz_okres', 'cz', 'cz010'): None,
('cz_okres', 'cz', 'cz020'): None,
('cz_okres', 'cz', 'cz031'): None,
('cz_okres', 'cz', 'cz032'): None,
('cz_okres', 'cz', 'cz041'): None,
('cz_okres', 'cz', 'cz042'): None,
('cz_okres', 'cz', 'cz051'): None,
('cz_okres', 'cz', 'cz052'): None,
('cz_okres', 'cz', 'cz053'): None,
('cz_okres', 'cz', 'cz064'): None,
('cz_okres', 'cz', 'cz071'): None,
('cz_okres', 'cz', 'cz063'): None,
('cz_okres', 'cz', 'cz072'): None,
('cz_okres', 'cz', 'cz080'): None,
},
mode=MODE_STRICT)
self.update()
def __init__(self):
# Only raw_data4.json is currently being updated,
# so won't download the others every day
URLBase.__init__(self,
output_dir=get_overseas_dir() / 'hk' / 'data',
urls_dict={
'T2_Data.json': URL(
'https://chp-dashboard.geodata.gov.hk/covid-19/T2_Data.json',
static_file=False
),
'cases.json': URL(
"""https://services8.arcgis.com/PXQv9PaDJHzt8rp0/arcgis/rest/services/Merge_Display_0227_View/FeatureServer/0/query?f=json&where=(Status = 'Existing' OR Status = 'History') AND (Case_no_ <> -1) AND (Status = 'History')&returnGeometry=true&spatialRel=esriSpatialRelIntersects&maxAllowableOffset=38&geometry={"xmin":12699553.627367351,"ymin":2543824.3013545386,"xmax":12719121.506608326,"ymax":2563392.180595517,"spatialReference":{"wkid":102100}}&geometryType=esriGeometryEnvelope&inSR=102100&outFields=*&outSR=102100&resultType=tile""".replace(' ', '%20'),
static_file=False
)
}
)
self.sdpf = StrictDataPointsFactory(
region_mappings={
('hk_district', 'hk', '黄大仙'): ('hk_district', 'hk', '黃大仙'),
('hk_district', 'hk', '荃灣區'): ('hk_district', 'hk', '荃灣'),
},
mode=MODE_STRICT
)
self.update()
def __init__(self):
URLBase.__init__(self,
output_dir=get_overseas_dir() / 'my' / 'esri_dash_data',
urls_dict={
'district_case.json': URL(
DISTRICT_CASE,
static_file=False
),
'sub_district_case.json': URL(
SUB_DISTRICT_CASE,
static_file=False
),
}
)
self.sdpf = StrictDataPointsFactory(
region_mappings={
('admin_1', 'my', 'johor'): ('admin_1', 'my', 'my-01'),
('admin_1', 'my', 'kedah'): ('admin_1', 'my', 'my-02'),
('admin_1', 'my', 'kelantan'): ('admin_1', 'my', 'my-03'),
('admin_1', 'my', 'kuala lumpur'): ('admin_1', 'my', 'my-14'),
('admin_1', 'my', 'labuan'): ('admin_1', 'my', 'my-15'),
('admin_1', 'my', 'melaka'): ('admin_1', 'my', 'my-04'),
('admin_1', 'my', 'my-01'): ('admin_1', 'my', 'my-01'),
('admin_1', 'my', 'my-02'): ('admin_1', 'my', 'my-02'),
('admin_1', 'my', 'my-03'): ('admin_1', 'my', 'my-03'),
('admin_1', 'my', 'my-04'): ('admin_1', 'my', 'my-04'),
('admin_1', 'my', 'my-05'): ('admin_1', 'my', 'my-05'),
('admin_1', 'my', 'my-06'): ('admin_1', 'my', 'my-06'),
('admin_1', 'my', 'my-07'): ('admin_1', 'my', 'my-07'),
('admin_1', 'my', 'my-08'): ('admin_1', 'my', 'my-08'),
('admin_1', 'my', 'my-09'): ('admin_1', 'my', 'my-09'),
('admin_1', 'my', 'my-10'): ('admin_1', 'my', 'my-10'),
('admin_1', 'my', 'my-11'): ('admin_1', 'my', 'my-11'),
('admin_1', 'my', 'my-12'): ('admin_1', 'my', 'my-12'),
('admin_1', 'my', 'my-13'): ('admin_1', 'my', 'my-13'),
('admin_1', 'my', 'my-14'): ('admin_1', 'my', 'my-14'),
('admin_1', 'my', 'my-15'): ('admin_1', 'my', 'my-15'),
('admin_1', 'my', 'my-16'): ('admin_1', 'my', 'my-16'),
('admin_1', 'my', 'negeri sembilan'): ('admin_1', 'my', 'my-05'),
('admin_1', 'my', 'pahang'): ('admin_1', 'my', 'my-06'),
('admin_1', 'my', 'perak'): ('admin_1', 'my', 'my-08'),
('admin_1', 'my', 'perlis'): ('admin_1', 'my', 'my-09'),
('admin_1', 'my', 'pulau pinang'): ('admin_1', 'my', 'my-07'),
('admin_1', 'my', 'putrajaya'): ('admin_1', 'my', 'my-16'),
('admin_1', 'my', 'sabah'): ('admin_1', 'my', 'my-12'),
('admin_1', 'my', 'sarawak'): ('admin_1', 'my', 'my-13'),
('admin_1', 'my', 'selangor'): ('admin_1', 'my', 'my-10'),
('admin_1', 'my', 'terengganu'): ('admin_1', 'my', 'my-11'),
('admin_1', 'my', 'tiada data'): None,
('my_district', 'my', 'alor gajah'): ('my_district', 'my', 'alor gajah'),
('my_district', 'my', 'asajaya'): ('my_district', 'my', 'asajaya'),
('my_district', 'my', 'bachok'): ('my_district', 'my', 'bachok'),
('my_district', 'my', 'baling'): ('my_district', 'my', 'baling'),
('my_district', 'my', 'bandar baharu'): ('my_district', 'my', 'bandar baharu'),
('my_district', 'my', 'bandar bahru'): ('my_district', 'my', 'bandar baharu'),
('my_district', 'my', 'bandar tun razak'): None,
('my_district', 'my', 'bangsar'): None,
('my_district', 'my', 'barat daya'): ('my_district', 'my', 'barat daya'),
('my_district', 'my', 'batang padang'): ('my_district', 'my', 'batang padang'),
('my_district', 'my', 'batu'): None,
('my_district', 'my', 'batu pahat'): ('my_district', 'my', 'batu pahat'),
('my_district', 'my', 'bau'): ('my_district', 'my', 'bau'),
('my_district', 'my', 'beaufort'): ('my_district', 'my', 'beaufort'),
('my_district', 'my', 'bentong'): ('my_district', 'my', 'bentong'),
('my_district', 'my', 'bera'): ('my_district', 'my', 'bera'),
('my_district', 'my', 'besut'): ('my_district', 'my', 'besut'),
('my_district', 'my', 'betong'): ('my_district', 'my', 'betong'),
('my_district', 'my', 'bintulu'): ('my_district', 'my', 'bintulu'),
('my_district', 'my', 'brickfields'): None,
('my_district', 'my', 'bukit damansara'): None,
('my_district', 'my', 'bukit jalil'): None,
('my_district', 'my', 'bukit bintang'): None,
('my_district', 'my', 'cameron highlands'): ('my_district', 'my', 'cameron highlands'),
('my_district', 'my', 'cheras'): None,
('my_district', 'my', 'dalat'): ('my_district', 'my', 'dalat'),
('my_district', 'my', 'daro'): ('my_district', 'my', 'daro'),
('my_district', 'my', 'dungun'): ('my_district', 'my', 'dungun'),
('my_district', 'my', 'duta'): None,
('my_district', 'my', 'gombak'): ('my_district', 'my', 'gombak'),
('my_district', 'my', 'gua musang'): ('my_district', 'my', 'gua musang'),
('my_district', 'my', 'hilir perak'): ('my_district', 'my', 'hilir perak'),
('my_district', 'my', 'hulu langat'): None,
('my_district', 'my', 'hulu perak'): None,
('my_district', 'my', 'hulu selangor'): None,
('my_district', 'my', 'hulu terengganu'): ('my_district', 'my', 'hulu terengganu'),
('my_district', 'my', 'ibu kota'): None,
('my_district', 'my', 'jasin'): ('my_district', 'my', 'jasin'),
('my_district', 'my', 'jelebu'): ('my_district', 'my', 'jelebu'),
('my_district', 'my', 'jeli'): ('my_district', 'my', 'jeli'),
('my_district', 'my', 'jempol'): ('my_district', 'my', 'jempol'),
('my_district', 'my', 'jerantut'): ('my_district', 'my', 'jerantut'),
('my_district', 'my', 'johor bahru'): ('my_district', 'my', 'johor bahru'),
('my_district', 'my', 'julau'): ('my_district', 'my', 'julau'),
('my_district', 'my', 'kampar'): ('my_district', 'my', 'kampar'),
('my_district', 'my', 'kampung bharu'): None,
('my_district', 'my', 'kangar'): None,
('my_district', 'my', 'kanowit'): ('my_district', 'my', 'kanowit'),
('my_district', 'my', 'kapit'): ('my_district', 'my', 'kapit'),
('my_district', 'my', 'kemaman'): ('my_district', 'my', 'kemaman'),
('my_district', 'my', 'keningau'): ('my_district', 'my', 'keningau'),
('my_district', 'my', 'kepong'): None,
('my_district', 'my', 'kerian'): ('my_district', 'my', 'kerian'),
('my_district', 'my', 'kinabatangan'): ('my_district', 'my', 'kinabatangan'),
('my_district', 'my', 'kinta'): ('my_district', 'my', 'kinta'),
('my_district', 'my', 'klang'): ('my_district', 'my', 'klang'),
('my_district', 'my', 'kluang'): ('my_district', 'my', 'kluang'),
('my_district', 'my', 'kota belud'): ('my_district', 'my', 'kota belud'),
('my_district', 'my', 'kota bharu'): ('my_district', 'my', 'kota bharu'),
('my_district', 'my', 'kota kinabalu'): ('my_district', 'my', 'kota kinabalu'),
('my_district', 'my', 'kota marudu'): ('my_district', 'my', 'kota marudu'),
('my_district', 'my', 'kota samarahan'): None,
('my_district', 'my', 'kota setar'): ('my_district', 'my', 'kota setar'),
('my_district', 'my', 'kota tinggi'): ('my_district', 'my', 'kota tinggi'),
('my_district', 'my', 'kuala kangsar'): ('my_district', 'my', 'kuala kangsar'),
('my_district', 'my', 'kuala krai'): ('my_district', 'my', 'kuala krai'),
('my_district', 'my', 'kuala langat'): ('my_district', 'my', 'kuala langat'),
('my_district', 'my', 'kuala lumpur'): None, # FIXME!!!!! ===================================================
('my_district', 'my', 'setiawangsa'): None,
('my_district', 'my', 'kuala muda'): ('my_district', 'my', 'kuala muda'),
('my_district', 'my', 'kuala pilah'): ('my_district', 'my', 'kuala pilah'),
('my_district', 'my', 'kuala selangor'): ('my_district', 'my', 'kuala selangor'),
('my_district', 'my', 'kuala terengganu'): ('my_district', 'my', 'kuala terengganu'),
('my_district', 'my', 'kuantan'): ('my_district', 'my', 'kuantan'),
('my_district', 'my', 'kubang pasu'): ('my_district', 'my', 'kubang pasu'),
('my_district', 'my', 'kuching'): ('my_district', 'my', 'kuching'),
('my_district', 'my', 'kudat'): ('my_district', 'my', 'kudat'),
('my_district', 'my', 'kulai'): None,
('my_district', 'my', 'kulim'): ('my_district', 'my', 'kulim'),
('my_district', 'my', 'labuan'): None,
('my_district', 'my', 'lahad datu'): ('my_district', 'my', 'lahad datu'),
('my_district', 'my', 'langkawi'): ('my_district', 'my', 'langkawi'),
('my_district', 'my', 'larut, matang, selama'): None,
('my_district', 'my', 'lawas'): ('my_district', 'my', 'lawas'),
('my_district', 'my', 'lembah pantai'): None,
('my_district', 'my', 'limbang'): ('my_district', 'my', 'limbang'),
('my_district', 'my', 'lipis'): ('my_district', 'my', 'lipis'),
('my_district', 'my', 'lubok antu'): ('my_district', 'my', 'lubok antu'),
('my_district', 'my', 'machang'): ('my_district', 'my', 'machang'),
('my_district', 'my', 'manjalara'): None,
('my_district', 'my', 'manjung'): None,
('my_district', 'my', 'maran'): ('my_district', 'my', 'maran'),
('my_district', 'my', 'marang'): ('my_district', 'my', 'marang'),
('my_district', 'my', 'matu'): ('my_district', 'my', 'matu'),
('my_district', 'my', 'melaka tengah'): ('my_district', 'my', 'melaka tengah'),
('my_district', 'my', 'mersing'): ('my_district', 'my', 'mersing'),
('my_district', 'my', 'miri'): ('my_district', 'my', 'miri'),
('my_district', 'my', 'mont kiara'): None,
('my_district', 'my', 'muar'): ('my_district', 'my', 'muar'),
('my_district', 'my', 'mukah'): ('my_district', 'my', 'mukah'),
('my_district', 'my', 'nabawan'): ('my_district', 'my', 'nabawan'),
('my_district', 'my', 'padang terap'): ('my_district', 'my', 'padang terap'),
('my_district', 'my', 'pakan'): ('my_district', 'my', 'pakan'),
('my_district', 'my', 'pandan'): None,
('my_district', 'my', 'pantai barat utara'): None,
('my_district', 'my', 'papar'): ('my_district', 'my', 'papar'),
('my_district', 'my', 'pasir mas'): ('my_district', 'my', 'pasir mas'),
('my_district', 'my', 'pasir puteh'): ('my_district', 'my', 'pasir puteh'),
('my_district', 'my', 'pekan'): ('my_district', 'my', 'pekan'),
('my_district', 'my', 'penampang'): ('my_district', 'my', 'penampang'),
('my_district', 'my', 'pendang'): ('my_district', 'my', 'pendang'),
('my_district', 'my', 'perak tengah'): ('my_district', 'my', 'perak tengah'),
('my_district', 'my', 'petaling'): ('my_district', 'my', 'petaling'),
('my_district', 'my', 'pitas'): ('my_district', 'my', 'pitas'),
('my_district', 'my', 'pokok sena'): ('my_district', 'my', 'pokok sena'),
('my_district', 'my', 'pontian'): ('my_district', 'my', 'pontian'),
('my_district', 'my', 'port dickson'): ('my_district', 'my', 'port dickson'),
('my_district', 'my', 'pudu'): None,
('my_district', 'my', 'putatan'): ('my_district', 'my', 'putatan'),
('my_district', 'my', 'putrajaya'): None,
('my_district', 'my', 'ranau'): ('my_district', 'my', 'ranau'),
('my_district', 'my', 'raub'): ('my_district', 'my', 'raub'),
('my_district', 'my', 'rembau'): ('my_district', 'my', 'rembau'),
('my_district', 'my', 'rompin'): ('my_district', 'my', 'rompin'),
('my_district', 'my', 'sabak bernam'): ('my_district', 'my', 'sabak bernam'),
('my_district', 'my', 'samarahan'): ('my_district', 'my', 'samarahan'),
('my_district', 'my', 'sandakan'): ('my_district', 'my', 'sandakan'),
('my_district', 'my', 'saratok'): ('my_district', 'my', 'saratok'),
('my_district', 'my', 'sarikei'): ('my_district', 'my', 'sarikei'),
('my_district', 'my', 'seberang perai selatan'): None,
('my_district', 'my', 'seberang perai tengah'): None,
('my_district', 'my', 'seberang perai utara'): None,
('my_district', 'my', 'segamat'): ('my_district', 'my', 'segamat'),
('my_district', 'my', 'semporna'): ('my_district', 'my', 'semporna'),
('my_district', 'my', 'sentul'): None,
('my_district', 'my', 'sepang'): ('my_district', 'my', 'sepang'),
('my_district', 'my', 'seputeh'): None,
('my_district', 'my', 'seremban'): ('my_district', 'my', 'seremban'),
('my_district', 'my', 'serian'): ('my_district', 'my', 'serian'),
('my_district', 'my', 'setiu'): ('my_district', 'my', 'setiu'),
('my_district', 'my', 'sibu'): ('my_district', 'my', 'sibu'),
('my_district', 'my', 'sik'): ('my_district', 'my', 'sik'),
('my_district', 'my', 'simunjan'): ('my_district', 'my', 'simunjan'),
('my_district', 'my', 'sipitang'): ('my_district', 'my', 'sipitang'),
('my_district', 'my', 'song'): ('my_district', 'my', 'song'),
('my_district', 'my', 'sri aman'): ('my_district', 'my', 'sri aman'),
('my_district', 'my', 'sri petaling'): None,
('my_district', 'my', 'tambunan'): ('my_district', 'my', 'tambunan'),
('my_district', 'my', 'tampin'): ('my_district', 'my', 'tampin'),
('my_district', 'my', 'tanah merah'): ('my_district', 'my', 'tanah merah'),
('my_district', 'my', 'tangkak'): None,
('my_district', 'my', 'tatau'): ('my_district', 'my', 'tatau'),
('my_district', 'my', 'tawau'): ('my_district', 'my', 'tawau'),
('my_district', 'my', 'temerloh'): ('my_district', 'my', 'temerloh'),
('my_district', 'my', 'tenom'): ('my_district', 'my', 'tenom'),
('my_district', 'my', 'timur laut'): ('my_district', 'my', 'timur laut'),
('my_district', 'my', 'titiwangsa'): None,
('my_district', 'my', 'tuaran'): ('my_district', 'my', 'tuaran'),
('my_district', 'my', 'tumpat'): ('my_district', 'my', 'tumpat'),
('my_district', 'my', 'wangsa maju'): None,
('my_district', 'my', 'yan'): ('my_district', 'my', 'yan'),
('my_district', 'my', 'kuala nerus'): None,
('my_district', 'my', 'telupid'): None,
('my_district', 'my', 'segambut'): None,
},
mode=MODE_STRICT
)
self.update()
def __init__(self):
GithubRepo.__init__(self,
output_dir=get_overseas_dir() / 'lk' / 'covid19-srilankan-data',
github_url='https://github.com/arimacdev/covid19-srilankan-data')
self.sdpf = StrictDataPointsFactory(mode=MODE_STRICT)
self.update()
def __init__(self):
GithubRepo.__init__(
self,
output_dir=get_overseas_dir() / 'us_nytimes' / 'covid-19-data',
github_url='https://github.com/nytimes/covid-19-data')
self.sdpf = StrictDataPointsFactory(region_mappings={
('us_county', 'alaska', 'fairbanks north star borough'):
None,
('us_county', 'alaska', 'ketchikan gateway borough'):
None,
('us_county', 'alaska', 'kenai peninsula borough'):
None,
('us_county', 'missouri', 'kansas city'):
None,
('us_county', 'alaska', 'juneau city and borough'):
None,
('us_county', 'alaska', 'matanuska-susitna borough'):
None,
('us_county', 'alaska', 'yukon-koyukuk census area'):
None,
('us_county', 'alaska', 'southeast fairbanks census area'):
None,
('us_county', 'alaska', 'petersburg borough'):
None,
('us_county', 'alaska', 'bethel census area'):
None,
('us_county', 'alaska', 'prince of wales-hyder census area'):
None,
('us_county', 'alaska', 'nome census area'):
None,
('us_county', 'alaska', 'kodiak island borough'):
None,
('us_county', 'alaska', 'sitka city and borough'):
None,
('us_county', 'puerto rico', 'anasco'):
None,
('us_county', 'puerto rico', 'bayamon'):
None,
('us_county', 'puerto rico', 'canovanas'):
None,
('us_county', 'puerto rico', 'catano'):
None,
('us_county', 'puerto rico', 'comerio'):
None,
('us_county', 'puerto rico', 'guanica'):
None,
('us_county', 'puerto rico', 'juana diaz'):
None,
('us_county', 'puerto rico', 'las marias'):
None,
('us_county', 'puerto rico', 'loiza'):
None,
('us_county', 'puerto rico', 'manati'):
None,
('us_county', 'puerto rico', 'mayaguez'):
None,
('us_county', 'puerto rico', 'penuelas'):
None,
('us_county', 'puerto rico', 'rincon'):
None,
('us_county', 'puerto rico', 'rio grande'):
None,
('us_county', 'puerto rico', 'san german'):
None,
('us_county', 'puerto rico', 'san sebastian'):
None,
('us_county', 'alaska', 'valdez-cordova census area'):
None,
('us_county', 'alaska', 'northwest arctic borough'):
None,
('us_county', 'alaska', 'north slope borough'):
None,
('us_county', 'alaska', 'bristol bay borough'):
None,
('us_county', 'alaska', 'dillingham census area'):
None,
('us_county', 'alaska', 'aleutians west census area'):
None,
('us_county', 'alaska', 'lake and peninsula borough'):
None,
('us_county', 'alaska', 'wrangell city and borough'):
None,
('us_county', 'alaska', 'aleutians east borough'):
None,
('us_county', 'alaska', 'haines borough'):
None,
('us_county', 'alaska', 'denali borough'):
None,
('us_county', 'alaska', 'kusilvak census area'):
None,
('us_county', 'alaska', 'skagway municipality'):
None,
('us_county', 'missouri', 'joplin'):
None,
('admin_1', 'us', 'puerto rico'):
None,
('admin_1', 'us', 'virgin islands'):
None,
('admin_1', 'us', 'guam'):
None,
('admin_1', 'us', 'northern mariana islands'):
None,
},
mode=MODE_STRICT)
self.update()
def _get_positive_by_department(self, date,
totals, added_totals,
tests, added_tests):
out = DataPointMerger()
base_path = get_overseas_dir() / 'fr' / 'esridata' / date
for fnam in sorted(listdir(base_path)):
r = self.sdpf()
with open(base_path / fnam, 'r', encoding='utf-8') as f:
data = json.loads(f.read())
print(base_path, fnam, data)
for property in data['features']:
attributes = property['attributes']
print(property)
date = self.convert_date(attributes['Jour'])
try:
region_child = 'FR-%02d' % int(attributes['CODE_DEPT'])
except ValueError:
# e.g. 2A
region_child = 'FR-%s' % attributes['CODE_DEPT']
for datatype, value in (
(DataTypes.STATUS_HOSPITALIZED, attributes['Hospitalisation_T']),
#(DataTypes.FIXME, attributes['Hospitalisation_H']),
#(DataTypes.FIXME, attributes['Hospitalisation_F']),
(DataTypes.STATUS_ICU, attributes['Reanimation_T']),
#(DataTypes.FIXME, attributes['Reanimation_H']),
#(DataTypes.FIXME, attributes['Reanimation_F']),
(DataTypes.STATUS_DEATHS, attributes['Deces_T']),
#(DataTypes.FIXME, attributes['Deces_H']),
#(DataTypes.FIXME, attributes['Deces_F']),
(DataTypes.NEW, attributes['Tests_Viro_P']),
#(DataTypes.TESTS_TOTAL, attributes['Tests_Viro_T']) # FIXME: This is new tests!
):
if value is None:
continue
r.append(
region_schema=Schemas.ADMIN_1,
region_parent='FR',
region_child=region_child,
datatype=datatype,
value=value,
date_updated=date,
source_url=self.SOURCE_URL
)
# I don't think Nbre_Cas_Confirmes is ever not None
assert attributes['Nbre_Cas_Confirmes'] is None, attributes
if attributes['Tests_Viro_P'] and not date in added_totals[region_child]:
added_totals[region_child].add(date)
totals[region_child] += attributes['Tests_Viro_P']# or attributes['Nbre_Cas_Confirmes'])
r.append(
region_schema=Schemas.ADMIN_1,
region_parent='FR',
region_child=region_child,
datatype=DataTypes.TOTAL,
value=totals[region_child],
date_updated=date,
source_url=self.SOURCE_URL
)
if attributes['Tests_Viro_T'] and not date in added_tests[region_child]:
added_tests[region_child].add(date)
tests[region_child] += attributes['Tests_Viro_T']
r.append(
region_schema=Schemas.ADMIN_1,
region_parent='FR',
region_child=region_child,
datatype=DataTypes.TESTS_TOTAL,
value=tests[region_child],
date_updated=date,
source_url=self.SOURCE_URL
)
out.extend(r)
return out
def __init__(self):
# Only raw_data4.json is currently being updated,
# so won't download the others every day
URLBase.__init__(
self,
output_dir=get_overseas_dir() / 'co' / 'data',
urls_dict={
'co_data.csv':
URL('https://proxy.hxlstandard.org/data.csv?dest=data_edit&tagger-match-all=on&tagger-01-header=id+de+caso&tagger-01-tag=%23meta+%2Bid&tagger-02-header=fecha+de+notificacion&tagger-02-tag=%23date+%2Bnotification&tagger-03-header=codigo+divipola&tagger-03-tag=%23code&tagger-04-header=ciudad+de+ubicacion&tagger-04-tag=%23adm3+%2Bname&tagger-05-header=departamento+o+distrito&tagger-05-tag=%23adm2+%2Bname&tagger-06-header=atencion&tagger-06-tag=%23indicator+%2Binfected+%2Btype&tagger-07-header=edad&tagger-07-tag=%23indicator+%2Binfected+%2Bage&tagger-08-header=sexo&tagger-08-tag=%23indicator+%2Binfected+%2Bsex&tagger-10-header=estado&tagger-10-tag=%23indicator+%2Binfected+%2Bstatus&tagger-11-header=pais+de+procedencia&tagger-11-tag=%23indicator+%2Binfected+%2Borigin&tagger-13-header=fecha+de+muerte&tagger-13-tag=%23date+%2Breported+%2Bdeath&tagger-14-header=fecha+diagnostico&tagger-14-tag=%23date+%2Breported+%2Bnotification&tagger-15-header=fecha+recuperado&tagger-15-tag=%23date+%2Breported+%2Brecovered&tagger-16-header=fecha+reporte+web&tagger-16-tag=%23date+%2Breported&header-row=1&url=https%3A%2F%2Fwww.datos.gov.co%2Fapi%2Fviews%2Fgt2j-8ykr%2Frows.csv%3FaccessType%3DDOWNLOAD',
static_file=False),
})
self.sdpf = StrictDataPointsFactory(
region_mappings={
('admin_1', 'co', 'co-dc'):
('MERGE', 'admin_1', 'co', 'co-cun'),
('admin_1', 'co', 'co-cun'):
('MERGE', 'admin_1', 'co', 'co-cun'),
('admin_1', 'co', 'cartagena d.t. y c.'):
('MERGE', 'admin_1', 'co', 'co-bol'),
('admin_1', 'co', 'cartagena'): ('MERGE', 'admin_1', 'co',
'co-bol'),
('admin_1', 'co', 'barranquilla d.e.'): ('MERGE', 'admin_1',
'co', 'co-atl'),
('admin_1', 'co', 'barranquilla'): ('MERGE', 'admin_1', 'co',
'co-atl'),
('admin_1', 'co', 'santa marta d.t. y c.'):
('MERGE', 'admin_1', 'co', 'co-atl'),
('admin_1', 'co', 'santa marta'): ('MERGE', 'admin_1', 'co',
'co-atl'),
('admin_1', 'co', 'archipiélago de san andrés providencia y santa catalina'):
('MERGE', 'admin_1', 'co', 'co-atl'),
('admin_1', 'co', 'buenaventura d.e.'): ('MERGE', 'admin_1',
'co', 'co-vac'),
('admin_1', 'co', 'buenaventura'): ('MERGE', 'admin_1', 'co',
'co-vac'),
('admin_1', 'co', 'norte santander'): ('MERGE', 'admin_1',
'co', 'co-vac'),
('co_municipality', 'cartagena d.t. y c.', 'cartagena de indias'):
('co_municipality', 'co-bol', 'cartagena de indias'),
('co_municipality', 'norte de santander', 'cúcuta'):
None,
('co_municipality', 'barranquilla d.e.', 'barranquilla'):
('co_municipality', 'co-atl', 'barranquilla'),
('co_municipality', 'santa marta d.t. y c.', 'santa marta'):
None,
('co_municipality', 'archipiélago de san andrés providencia y santa catalina', 'san andrés'):
None,
('co_municipality', 'archipiélago de san andrés providencia y santa catalina', 'providencia'):
None,
('co_municipality', 'buenaventura d.e.', 'buenaventura'):
('co_municipality', 'co-vac', 'buenaventura'),
('co_municipality', 'co-dc', 'bogotá d.c.'):
None, # FIXME!!!!
('co_municipality', 'cundinamarca', 'villa de de ubaté'):
None,
('co_municipality', 'nariño', 'cuaspúd'):
None,
('co_municipality', 'nariño', 'ancuyá'):
None,
('co_municipality', 'cauca', 'piendamó'):
None,
('co_municipality', 'cauca', 'guapí'):
None,
('co_municipality', 'cauca', 'sotara'):
None,
('co_municipality', 'vaupés', 'mitú'):
None,
('co_municipality', 'vaupés', 'yavaraté'):
None,
('co_municipality', 'vaupés', 'taraira'):
None,
('co_municipality', 'chocó', 'el cantón san pablo'):
None,
('co_municipality', 'chocó', 'el litoral san juan'):
None,
('co_municipality', 'sucre', 'tolú viejo'):
None,
('co_municipality', 'sucre', 'coloso'):
None,
('co_municipality', 'antioquia', 'san pedro los milagros'):
None,
('co_municipality', 'antioquia', 'san josé la montaña'):
None,
('co_municipality', 'meta', 'san luis de cubarral'):
None,
('co_municipality', 'santander', 'el carmen de chucurí'):
None,
('co_municipality', 'vaupés', 'carurú'):
None,
('co_municipality', 'guainía', 'barranco minas'):
None,
},
mode=MODE_STRICT)
self.update()
def __init__(self):
GithubRepo.__init__(self,
output_dir=get_overseas_dir() / 'nz' / 'nz-covid19-data',
github_url='https://github.com/philiprenich/nz-covid19-data')
self.sdpf = StrictDataPointsFactory()
self.update()
def get_datapoints(self):
r = []
dpm = DataPointMerger()
for date in sorted(listdir(get_overseas_dir() / 'uk' / 'gov-api')):
r.extend(self._get_datapoints(date, dpm))
return r
def __init__(self):
# Only raw_data4.json is currently being updated,
# so won't download the others every day
URLBase.__init__(
self,
output_dir=get_overseas_dir() / 'ly' / 'data',
urls_dict={
'ly_data.csv':
URL(
'https://docs.google.com/spreadsheets/d/e/'
'2PACX-1vQQWJZmGZJfUm22CPWoeW6rSS7Xh4K54r4A8RlN214ZCIPBUBOug3UbxFPrbiT3FQic6HS8wGdUhv3f/'
'pub?output=csv',
static_file=False)
})
self.sdpf = StrictDataPointsFactory(
region_mappings={
('admin_1', 'ly', 'اجدابيا'): None, # Ajdabiya, Al Wahat
('admin_1', 'ly', 'البريقة'): None, # Brega, Al Wahat
('admin_1', 'ly', 'البيضاء'): None, # Bayda, Jabal al Akhdar
('admin_1', 'ly', 'الجفارة'): None, # Jafara
('admin_1', 'ly', 'الجميل'): None, # Aljmail, Nuqat al Khams
('admin_1', 'ly', 'الخمس'): None, # Al-Khums, Murqub
('admin_1', 'ly', 'الرجبان'): None, # Alrujban, ???
('admin_1', 'ly', 'الرحيبات'): None, # Alruhaibat, ???
('admin_1', 'ly', 'الرياينة'): None, # Alriyayna, ???
('admin_1', 'ly', 'الزاوية'): None, # Az-Zāwiyah, ???
('admin_1', 'ly', 'الزنتان'):
None, # Alzintan, Jabal al Gharbi
('admin_1', 'ly', 'العجيلات'): None, # Ajaylat, Nuqat al Khams
('admin_1', 'ly', 'العزيزية'): None, # ʽAziziya, Jafara
('admin_1', 'ly', 'القلعة'): None,
('admin_1', 'ly', 'الكفرة'): None, # Kufra District
('admin_1', 'ly', 'المحروقة'): None,
('admin_1', 'ly', 'المرج'): None, # Marj, Marj
('admin_1', 'ly', 'بني وليد'):
None, # Bani Walid, Misrata District
('admin_1', 'ly', 'جادو'): None,
('admin_1', 'ly', 'جنزور'):
None, # Janzur, Janzour/Greater Tripoli???
('admin_1', 'ly', 'درنة'): None, # Derna, Derna
('admin_1', 'ly', 'رقدالين'): None,
('admin_1', 'ly', 'زلطن'): None,
('admin_1', 'ly', 'زليتن'): None,
('admin_1', 'ly', 'زوارة'): None,
('admin_1', 'ly', 'سبها'): None,
('admin_1', 'ly', 'سرت'): None,
('admin_1', 'ly', 'صبراتة'): None,
('admin_1', 'ly', 'صرمان'): None,
('admin_1', 'ly', 'طبرق'): None,
('admin_1', 'ly', 'طرابلس'): None,
('admin_1', 'ly', 'غدامس'): None,
('admin_1', 'ly', 'غريان'): None,
('admin_1', 'ly', 'قصر الاخيار'): None,
('admin_1', 'ly', 'قصر بن غشير'): None,
('admin_1', 'ly', 'مزدة'): None,
('admin_1', 'ly', 'مسلاتة'): None,
('admin_1', 'ly', 'مصراتة'): None,
('admin_1', 'ly', 'نالوت'): None,
('admin_1', 'ly', 'هراوة'): None,
('admin_1', 'ly', 'وادي الشاطيء'): None,
('admin_1', 'ly', 'يفرن'): None,
('admin_1', 'ly', 'سوسة'): None,
('admin_1', 'ly', 'تيجي'): None,
('admin_1', 'ly', 'امساعد'): None,
('admin_1', 'ly', 'الزهرة'): None,
('admin_1', 'ly', 'السبيعة'): None,
('admin_1', 'ly', 'تندميرة'): None,
('admin_1', 'ly', 'بن جواد'): None,
('admin_1', 'ly', 'الأبيار'): None,
('admin_1', 'ly', 'كاباو'): None,
('admin_1', 'ly', 'الجفرة'): None,
('admin_1', 'ly', 'ككلة'): None,
('admin_1', 'ly', 'الشويرف'): None,
('admin_1', 'ly', 'ترهونة'): None,
},
mode=MODE_STRICT)
self.update()
def __download_dataset(self, area_type):
"""
Extracts paginated data by requesting all of the pages
and combining the results.
"""
date = datetime.datetime.now().strftime('%Y_%m_%d')
dir_ = get_overseas_dir() / 'uk' / 'gov-api' / date
if not exists(dir_):
makedirs(dir_)
path = dir_ / f'{area_type}.json'
if exists(path):
# Don't download if already downloaded!
return
endpoint = "https://api.coronavirus.data.gov.uk/v1/data"
structure = {
"date": "date",
"name": "areaName",
"code": "areaCode",
"daily": "newCasesBySpecimenDate",
"cumulative": "cumCasesBySpecimenDate",
"deathsDaily": "newDeaths28DaysByPublishDate",
"deathsCumulative": "cumDeaths28DaysByPublishDate",
"cumTestsByPublishDate": "cumTestsByPublishDate",
"newTestsByPublishDate": "newTestsByPublishDate",
"newAdmissions": "newAdmissions",
"cumAdmissions": "cumAdmissions",
"cumAdmissionsByAge": "cumAdmissionsByAge",
"hospitalCases": "hospitalCases",
"covidOccupiedMVBeds": "covidOccupiedMVBeds",
"maleCases": "maleCases",
"femaleCases": "femaleCases"
}
api_params = {
"filters": f"areaType={area_type}",
"structure": dumps(structure, separators=(",", ":")),
"format": "json"
}
data = []
page_number = 1
while True:
# Adding page number to query params
api_params["page"] = page_number
print("getting:", endpoint, api_params)
response = get(endpoint, params=api_params, timeout=30)
if response.status_code >= HTTPStatus.BAD_REQUEST:
raise RuntimeError(f'Request failed: {response.text}')
elif response.status_code == HTTPStatus.NO_CONTENT:
break
current_data = response.json()
page_data = current_data['data']
data.extend(page_data)
#print(page_data)
# The "next" attribute in "pagination" will be `None`
# when we reach the end.
if current_data["pagination"]["next"] is None:
break
elif page_number > 100:
break
page_number += 1
with open(path, 'w', encoding='utf-8') as f:
f.write(dumps(data))
from covid_crawlers.se_asia.jp_tokyo_data.tokyo_pdf_stats_map import stats_map, CITY
_tokyo_cities_to_en = get_tokyo_cities_to_en_map()
def pq(*args, **kw):
for x in range(5):
try:
return _pq(*args, **kw)
except:
if x == 4:
raise
time.sleep(1)
PDFS_BASE_DIR = get_overseas_dir() / 'jp_city_data' / 'pdfs'
class ExtractFromTokyoPDF:
def download_pdfs(self, only_most_recent=True):
current_month = datetime.datetime.now().month
if only_most_recent:
months = [current_month]
else:
months = [
month for month in range(1, 13)
]
for month in months:
url = 'https://www.metro.tokyo.lg.jp/tosei/' \
def __init__(self):
GithubRepo.__init__(
self,
output_dir=get_overseas_dir() / 'my' / 'covid-19-malaysia',
github_url='https://github.com/ynshung/covid-19-malaysia')
self.sdpf = StrictDataPointsFactory(region_mappings={
('admin_0', '', 'malaysia'): ('admin_0', '', 'my'),
('admin_0', '', 'my'): ('admin_0', '', 'my'),
('admin_1', 'malaysia', 'johor'): ('admin_1', 'my', 'my-01'),
('admin_1', 'malaysia', 'kedah'): ('admin_1', 'my', 'my-02'),
('admin_1', 'malaysia', 'kelantan'): ('admin_1', 'my', 'my-03'),
('admin_1', 'malaysia', 'melaka'): ('admin_1', 'my', 'my-04'),
('admin_1', 'malaysia', 'negeri-sembilan'):
None,
('admin_1', 'malaysia', 'pahang'): ('admin_1', 'my', 'my-06'),
('admin_1', 'malaysia', 'perak'): ('admin_1', 'my', 'my-08'),
('admin_1', 'malaysia', 'perlis'): ('admin_1', 'my', 'my-09'),
('admin_1', 'malaysia', 'pulau-pinang'):
None,
('admin_1', 'malaysia', 'sabah'): ('admin_1', 'my', 'my-12'),
('admin_1', 'malaysia', 'sarawak'): ('admin_1', 'my', 'my-13'),
('admin_1', 'malaysia', 'selangor'): ('admin_1', 'my', 'my-10'),
('admin_1', 'malaysia', 'terengganu'): ('admin_1', 'my', 'my-11'),
('admin_1', 'malaysia', 'wp-kuala-lumpur'):
None,
('admin_1', 'malaysia', 'wp-labuan'):
None,
('admin_1', 'malaysia', 'wp-putrajaya'):
None,
('admin_1', 'my', 'my-01'): ('admin_1', 'my', 'my-01'),
('admin_1', 'my', 'my-02'): ('admin_1', 'my', 'my-02'),
('admin_1', 'my', 'my-03'): ('admin_1', 'my', 'my-03'),
('admin_1', 'my', 'my-04'): ('admin_1', 'my', 'my-04'),
('admin_1', 'my', 'my-06'): ('admin_1', 'my', 'my-06'),
('admin_1', 'my', 'my-08'): ('admin_1', 'my', 'my-08'),
('admin_1', 'my', 'my-09'): ('admin_1', 'my', 'my-09'),
('admin_1', 'my', 'my-10'): ('admin_1', 'my', 'my-10'),
('admin_1', 'my', 'my-11'): ('admin_1', 'my', 'my-11'),
('admin_1', 'my', 'my-12'): ('admin_1', 'my', 'my-12'),
('admin_1', 'my', 'my-13'): ('admin_1', 'my', 'my-13'),
('my_district', 'my', 'alor-gajah'):
None,
('my_district', 'my', 'asajaya'): ('my_district', 'my', 'asajaya'),
('my_district', 'my', 'bachok'): ('my_district', 'my', 'bachok'),
('my_district', 'my', 'baling'): ('my_district', 'my', 'baling'),
('my_district', 'my', 'bandar-bahru'):
None,
('my_district', 'my', 'barat-daya'):
None,
('my_district', 'my', 'batang-padang'):
None,
('my_district', 'my', 'batu-pahat'):
None,
('my_district', 'my', 'bau'): ('my_district', 'my', 'bau'),
('my_district', 'my', 'beaufort'):
('my_district', 'my', 'beaufort'),
('my_district', 'my', 'belaga'): ('my_district', 'my', 'belaga'),
('my_district', 'my', 'beluran'): ('my_district', 'my', 'beluran'),
('my_district', 'my', 'beluru'):
None,
('my_district', 'my', 'bentong'): ('my_district', 'my', 'bentong'),
('my_district', 'my', 'bera'): ('my_district', 'my', 'bera'),
('my_district', 'my', 'besut'): ('my_district', 'my', 'besut'),
('my_district', 'my', 'betong'): ('my_district', 'my', 'betong'),
('my_district', 'my', 'bintulu'): ('my_district', 'my', 'bintulu'),
('my_district', 'my', 'bukit-mabong'):
None,
('my_district', 'my', 'cameron'):
None,
('my_district', 'my', 'cheras'):
None,
('my_district', 'my', 'dalat'): ('my_district', 'my', 'dalat'),
('my_district', 'my', 'daro'): ('my_district', 'my', 'daro'),
('my_district', 'my', 'dungun'): ('my_district', 'my', 'dungun'),
('my_district', 'my', 'gombak'): ('my_district', 'my', 'gombak'),
('my_district', 'my', 'gua-musang'):
None,
('my_district', 'my', 'hillir-perak'):
None,
('my_district', 'my', 'hulu-langat'):
None,
('my_district', 'my', 'hulu-perak'):
None,
('my_district', 'my', 'hulu-selangor'):
None,
('my_district', 'my', 'hulu-terengganu'):
None,
('my_district', 'my', 'jasin'): ('my_district', 'my', 'jasin'),
('my_district', 'my', 'jelebu'): ('my_district', 'my', 'jelebu'),
('my_district', 'my', 'jeli'): ('my_district', 'my', 'jeli'),
('my_district', 'my', 'jempol'): ('my_district', 'my', 'jempol'),
('my_district', 'my', 'jerantut'):
('my_district', 'my', 'jerantut'),
('my_district', 'my', 'johor-bahru'):
None,
('my_district', 'my', 'julau'): ('my_district', 'my', 'julau'),
('my_district', 'my', 'kabong'):
None,
('my_district', 'my', 'kampar'): ('my_district', 'my', 'kampar'),
('my_district', 'my', 'kanowit'): ('my_district', 'my', 'kanowit'),
('my_district', 'my', 'kapit'): ('my_district', 'my', 'kapit'),
('my_district', 'my', 'kemaman'): ('my_district', 'my', 'kemaman'),
('my_district', 'my', 'keningau'):
('my_district', 'my', 'keningau'),
('my_district', 'my', 'kepong'):
None,
('my_district', 'my', 'kerian'): ('my_district', 'my', 'kerian'),
('my_district', 'my', 'kinabatangan'):
('my_district', 'my', 'kinabatangan'),
('my_district', 'my', 'kinta'): ('my_district', 'my', 'kinta'),
('my_district', 'my', 'klang'): ('my_district', 'my', 'klang'),
('my_district', 'my', 'kluang'): ('my_district', 'my', 'kluang'),
('my_district', 'my', 'kota-belud'):
None,
('my_district', 'my', 'kota-bharu'):
None,
('my_district', 'my', 'kota-kinabalu'):
None,
('my_district', 'my', 'kota-marudu'):
None,
('my_district', 'my', 'kota-samarahan'):
None,
('my_district', 'my', 'kota-setar'):
None,
('my_district', 'my', 'kota-tinggi'):
None,
('my_district', 'my', 'kuala-kangsar'):
None,
('my_district', 'my', 'kuala-krai'):
None,
('my_district', 'my', 'kuala-langat'):
None,
('my_district', 'my', 'kuala-muda'):
None,
('my_district', 'my', 'kuala-nerus'):
None,
('my_district', 'my', 'kuala-penyu'):
None,
('my_district', 'my', 'kuala-pilah'):
None,
('my_district', 'my', 'kuala-selangor'):
None,
('my_district', 'my', 'kuala-terengganu'):
None,
('my_district', 'my', 'kuantan'): ('my_district', 'my', 'kuantan'),
('my_district', 'my', 'kubang-pasu'):
None,
('my_district', 'my', 'kuching'): ('my_district', 'my', 'kuching'),
('my_district', 'my', 'kudat'): ('my_district', 'my', 'kudat'),
('my_district', 'my', 'kulai'):
None,
('my_district', 'my', 'kulim'): ('my_district', 'my', 'kulim'),
('my_district', 'my', 'kunak'): ('my_district', 'my', 'kunak'),
('my_district', 'my', 'lahad-datu'):
None,
('my_district', 'my', 'langkawi'): ('my_district', 'my',
'langkawi'),
('my_district', 'my', 'larut-matang-selama'):
None,
('my_district', 'my', 'lawas'): ('my_district', 'my', 'lawas'),
('my_district', 'my', 'lembah-pantai'):
None,
('my_district', 'my', 'limbang'): ('my_district', 'my', 'limbang'),
('my_district', 'my', 'lipis'): ('my_district', 'my', 'lipis'),
('my_district', 'my', 'lubok-antu'):
None,
('my_district', 'my', 'lundu'): ('my_district', 'my', 'lundu'),
('my_district', 'my', 'machang'): ('my_district', 'my', 'machang'),
('my_district', 'my', 'manjung'):
None,
('my_district', 'my', 'maran'): ('my_district', 'my', 'maran'),
('my_district', 'my', 'marang'): ('my_district', 'my', 'marang'),
('my_district', 'my', 'marudi'): ('my_district', 'my', 'marudi'),
('my_district', 'my', 'matu'): ('my_district', 'my', 'matu'),
('my_district', 'my', 'melaka-tengah'):
None,
('my_district', 'my', 'meradong'): ('my_district', 'my',
'meradong'),
('my_district', 'my', 'mersing'): ('my_district', 'my', 'mersing'),
('my_district', 'my', 'miri'): ('my_district', 'my', 'miri'),
('my_district', 'my', 'muallim'):
None,
('my_district', 'my', 'muar'): ('my_district', 'my', 'muar'),
('my_district', 'my', 'mukah'): ('my_district', 'my', 'mukah'),
('my_district', 'my', 'nabawan'): ('my_district', 'my', 'nabawan'),
('my_district', 'my', 'padang-terap'):
None,
('my_district', 'my', 'pakan'): ('my_district', 'my', 'pakan'),
('my_district', 'my', 'papar'): ('my_district', 'my', 'papar'),
('my_district', 'my', 'pasir-mas'):
None,
('my_district', 'my', 'pasir-putih'):
None,
('my_district', 'my', 'pekan'): ('my_district', 'my', 'pekan'),
('my_district', 'my', 'penampang'): ('my_district', 'my',
'penampang'),
('my_district', 'my', 'pendang'): ('my_district', 'my', 'pendang'),
('my_district', 'my', 'perak-tengah'):
None,
('my_district', 'my', 'petaling'): ('my_district', 'my',
'petaling'),
('my_district', 'my', 'pitas'): ('my_district', 'my', 'pitas'),
('my_district', 'my', 'pontian'): ('my_district', 'my', 'pontian'),
('my_district', 'my', 'port-dickson'):
None,
('my_district', 'my', 'pusa'):
None,
('my_district', 'my', 'putatan'): ('my_district', 'my', 'putatan'),
('my_district', 'my', 'ranau'): ('my_district', 'my', 'ranau'),
('my_district', 'my', 'raub'): ('my_district', 'my', 'raub'),
('my_district', 'my', 'rembau'): ('my_district', 'my', 'rembau'),
('my_district', 'my', 'rompin'): ('my_district', 'my', 'rompin'),
('my_district', 'my', 'sabak-bernam'):
None,
('my_district', 'my', 'sandakan'): ('my_district', 'my',
'sandakan'),
('my_district', 'my', 'saratok'): ('my_district', 'my', 'saratok'),
('my_district', 'my', 'sarikei'): ('my_district', 'my', 'sarikei'),
('my_district', 'my', 'sebauh'):
None,
('my_district', 'my', 'seberang-perai-selatan'):
None,
('my_district', 'my', 'seberang-perai-tengah'):
None,
('my_district', 'my', 'seberang-perai-utara'):
None,
('my_district', 'my', 'segamat'): ('my_district', 'my', 'segamat'),
('my_district', 'my', 'selangau'): ('my_district', 'my',
'selangau'),
('my_district', 'my', 'semporna'): ('my_district', 'my',
'semporna'),
('my_district', 'my', 'sepang'): ('my_district', 'my', 'sepang'),
('my_district', 'my', 'seremban'): ('my_district', 'my',
'seremban'),
('my_district', 'my', 'serian'): ('my_district', 'my', 'serian'),
('my_district', 'my', 'setiu'): ('my_district', 'my', 'setiu'),
('my_district', 'my', 'sibu'): ('my_district', 'my', 'sibu'),
('my_district', 'my', 'sik'): ('my_district', 'my', 'sik'),
('my_district', 'my', 'simunjan'): ('my_district', 'my',
'simunjan'),
('my_district', 'my', 'sipitang'): ('my_district', 'my',
'sipitang'),
('my_district', 'my', 'song'): ('my_district', 'my', 'song'),
('my_district', 'my', 'sri-aman'):
None,
('my_district', 'my', 'subis'):
None,
('my_district', 'my', 'tambunan'): ('my_district', 'my',
'tambunan'),
('my_district', 'my', 'tampin'): ('my_district', 'my', 'tampin'),
('my_district', 'my', 'tanah-merah'):
None,
('my_district', 'my', 'tangkak'):
None,
('my_district', 'my', 'tanjung-manis'):
None,
('my_district', 'my', 'tatau'): ('my_district', 'my', 'tatau'),
('my_district', 'my', 'tawau'): ('my_district', 'my', 'tawau'),
('my_district', 'my', 'tebedu'):
None,
('my_district', 'my', 'telung-usan'):
None,
('my_district', 'my', 'temerloh'): ('my_district', 'my',
'temerloh'),
('my_district', 'my', 'tenom'): ('my_district', 'my', 'tenom'),
('my_district', 'my', 'timur-laut'):
None,
('my_district', 'my', 'titiwangsa'):
None,
('my_district', 'my', 'tongod'): ('my_district', 'my', 'tongod'),
('my_district', 'my', 'tuaran'): ('my_district', 'my', 'tuaran'),
('my_district', 'my', 'tumpat'): ('my_district', 'my', 'tumpat'),
('my_district', 'my', 'yan'): ('my_district', 'my', 'yan')
},
mode=MODE_STRICT)
self.update()
def _get_datapoints(self):
r = self.sdpf()
for date in self.iter_nonempty_dirs(self.output_dir):
for province_dict in provinces:
if province_dict["value"] in ('86', '88', '710000', '810000'):
continue
path = get_overseas_dir(
) / 'cn' / 'data' / date / f'{province_dict["value"]}.json'
print(path)
with open(path, 'r', encoding='utf-8') as f:
json_data = f.read()
json_data = json_data.replace(
"dojo_request_script_callbacks.dojo_request_script0(",
"")
json_data = json_data.rstrip().rstrip(");")
data = json.loads(json_data)
# {"author":"geodata","city":"河南省","code":"410000","codeis":"410000",
# "country":"中国","createAt":1599211044000,"day":"2020-09-03",
# "id":"43030b8e44d35d00af7a25deb55524a4",
# "nconfirm":0,"ndead":0,"nheal":0,"nsuspect":0,"params":{},
# "province":"河南","publishAt":1599148740000,
# "tconfirm":1276,"tdead":22,"theal":1254,"tsuspect":0,
# "updateAt":1599211044000}
for item in data:
i_date = self.convert_date(item['day'])
r.append(
region_schema=Schemas.CN_CITY,
region_parent='CN', #item['province'],
region_child=item['city'],
datatype=DataTypes.TOTAL,
value=int(item[f'tconfirm']) + int(item[f'tsuspect']),
date_updated=i_date,
source_url=self.SOURCE_URL)
r.append(
region_schema=Schemas.CN_CITY,
region_parent='CN', #item['province'],
region_child=item['city'],
datatype=DataTypes.NEW,
value=int(item[f'nconfirm']) + int(item[f'nsuspect']),
date_updated=i_date,
source_url=self.SOURCE_URL)
for total_datatype, new_datatype, key_suffix in (
(DataTypes.CONFIRMED, DataTypes.CONFIRMED_NEW,
'confirm'),
(DataTypes.PROBABLE, DataTypes.PROBABLE_NEW,
'suspect'),
(DataTypes.STATUS_RECOVERED,
DataTypes.STATUS_RECOVERED_NEW, 'heal'),
(DataTypes.STATUS_DEATHS, DataTypes.STATUS_DEATHS_NEW,
'dead'),
):
r.append(
region_schema=Schemas.CN_CITY,
region_parent='CN', #item['province'],
region_child=item['city'],
datatype=total_datatype,
value=int(item[f'n{key_suffix}']),
date_updated=i_date,
source_url=self.SOURCE_URL)
r.append(
region_schema=Schemas.CN_CITY,
region_parent='CN', #item['province'],
region_child=item['city'],
datatype=new_datatype,
value=int(item[f't{key_suffix}']),
date_updated=i_date,
source_url=self.SOURCE_URL)
return r
def _get_datapoints(self):
# dateRep day month year cases deaths countriesAndTerritories geoId countryterritoryCode
# popData2019 continentExp Cumulative_number_for_14_days_of_COVID-19_cases_per_100000
#s
# 25/08/2020 25 8 2020 71 10 Afghanistan AF AFG 38041757 Asia 2.67074941
# 24/08/2020 24 8 2020 0 0 Afghanistan AF AFG 38041757 Asia 2.48411239
# 23/08/2020 23 8 2020 105 2 Afghanistan AF AFG 38041757 Asia 2.48411239
# 22/08/2020 22 8 2020 38 0 Afghanistan AF AFG 38041757 Asia 2.31061883
# 21/08/2020 21 8 2020 97 2 Afghanistan AF AFG 38041757 Asia 2.41576644
# 20/08/2020 20 8 2020 160 8 Afghanistan AF AFG 38041757 Asia 2.26855978
r = self.sdpf()
date = datetime.today().strftime('%Y_%m_%d')
df = pd.read_excel(get_overseas_dir() / 'world_eu_cdc' / 'data' /
date / 'world_data.xlsx',
engine="openpyxl",
parse_dates=False)
date_rep_out = []
for idx, item in df.iterrows():
if '/' in item['dateRep']:
dd, mm, yyyy = item['dateRep'].split('/')
date_rep_out.append(f'{yyyy}/{mm}/{dd}')
else:
date_rep_out.append(item['dateRep'])
df['dateRep'] = date_rep_out
df = df.sort_values('dateRep', axis=0)
prev_date = None
cases_cumulative = Counter()
deaths_cumulative = Counter()
for idx, item in df.iterrows():
date = self.convert_date(str(item['dateRep']).split()[0])
if item['geoId'] in ('BQ', 'JPG11668'):
continue
elif item['geoId'] == 'EL':
item['geoId'] = 'gr'
elif item['geoId'] == 'UK':
item['geoId'] = 'gb'
elif not isinstance(item['geoId'], str):
continue
if prev_date and prev_date > date:
raise Exception(prev_date, date)
prev_date = date
cases_cumulative[item['geoId']] += int(item['cases'])
deaths_cumulative[item['geoId']] += int(item['deaths'])
r.append(region_schema=Schemas.ADMIN_0,
region_parent='',
region_child=item['geoId'],
datatype=DataTypes.TOTAL,
value=cases_cumulative[item['geoId']],
date_updated=date,
source_url=self.SOURCE_URL)
r.append(region_schema=Schemas.ADMIN_0,
region_parent='',
region_child=item['geoId'],
datatype=DataTypes.STATUS_DEATHS,
value=deaths_cumulative[item['geoId']],
date_updated=date,
source_url=self.SOURCE_URL)
return r
def __init__(self):
GithubRepo.__init__(self,
output_dir=get_overseas_dir() / 'fr' / 'data',
github_url='https://github.com/opencovid19-fr/data')
self.sdpf = StrictDataPointsFactory(mode=MODE_STRICT)
self.update()
def __init__(self):
# Only raw_data4.json is currently being updated,
# so won't download the others every day
URLBase.__init__(
self,
output_dir=get_overseas_dir() / 'sn' / 'data',
urls_dict={
'by_district.csv':
URL(
'https://docs.google.com/spreadsheets/d/e/'
'2PACX-1vRj1sRWYmyZ2AznFdP5Dr98uZrzsMMudPBRIcMW8FdwAEy-'
'Hwq3PSPJYI12xTzLbA/pub?gid=1515611831&single=true&output=csv',
static_file=False),
'national_cases.csv':
URL(
'https://docs.google.com/spreadsheets/d/e/'
'2PACX-1vRj1sRWYmyZ2AznFdP5Dr98uZrzsMMudPBRIcMW8FdwAEy-'
'Hwq3PSPJYI12xTzLbA/pub?gid=708820609&single=true&output=csv',
static_file=False)
})
self.sdpf = StrictDataPointsFactory(
region_mappings={
('admin_1', 'sn', 'dakar-ouest'):
('MERGE', 'admin_1', 'sn', 'sn-dk'),
('admin_1', 'sn', 'dakar'):
('MERGE', 'admin_1', 'sn', 'sn-dk'),
('admin_1', 'sn', 'dakar-centre'):
('MERGE', 'admin_1', 'sn', 'sn-dk'),
('admin_1', 'sn', 'dakar-nord'):
('MERGE', 'admin_1', 'sn', 'sn-dk'),
('admin_1', 'sn', 'dakar-sud'): ('MERGE', 'admin_1', 'sn',
'sn-dk'),
('admin_1', 'sn', 'mbao'): ('MERGE', 'admin_1', 'sn', 'sn-dk'),
('admin_1', 'sn', 'guédiawaye'): ('MERGE', 'admin_1', 'sn',
'sn-dk'),
('admin_1', 'sn', 'yeumbeul'): ('MERGE', 'admin_1', 'sn',
'sn-dk'),
('admin_1', 'sn', 'rufisque'): ('MERGE', 'admin_1', 'sn',
'sn-dk'),
('admin_1', 'sn', 'diamniadio'): ('MERGE', 'admin_1', 'sn',
'sn-dk'),
('admin_1', 'sn', 'pikine'): ('MERGE', 'admin_1', 'sn',
'sn-dk'),
('admin_1', 'sn', 'sangalkam'): ('MERGE', 'admin_1', 'sn',
'sn-dk'),
('admin_1', 'sn', 'keur massar'): ('MERGE', 'admin_1', 'sn',
'sn-dk'),
('admin_1', 'sn', 'guediawaye'): ('MERGE', 'admin_1', 'sn',
'sn-dk'),
('admin_1', 'sn', 'touba'): ('MERGE', 'admin_1', 'sn',
'sn-db'),
('admin_1', 'sn', 'mbour'): ('MERGE', 'admin_1', 'sn',
'sn-th'),
('admin_1', 'sn', 'popenguine'): ('MERGE', 'admin_1', 'sn',
'sn-th'),
('admin_1', 'sn', 'oussouye'): ('MERGE', 'admin_1', 'sn',
'sn-zg'),
('admin_1', 'sn', 'saint louis'): ('MERGE', 'admin_1', 'sn',
'sn-sl'),
('admin_1', 'sn', 'richard toll'): ('MERGE', 'admin_1', 'sn',
'sn-sl'),
('admin_1', 'sn', 'non déterminé'): ('MERGE', 'admin_1', 'sn',
'unknown'),
('admin_1', 'sn', ''): ('MERGE', 'admin_1', 'sn', 'unknown'),
('admin_1', 'sn', 'velingara'): ('MERGE', 'admin_1', 'sn',
'sn-kd'),
('admin_1', 'sn', 'goudiry'): ('MERGE', 'admin_1', 'sn',
'sn-tc'),
('admin_1', 'sn', 'keur massar'): ('MERGE', 'admin_1', 'sn',
'sn-tc'),
('admin_1', 'sn', 'sakal'): ('MERGE', 'admin_1', 'sn',
'sn-lg'),
('admin_1', 'sn', 'mbacke'): ('MERGE', 'admin_1', 'sn',
'sn-db'),
('admin_1', 'sn', 'tivaouane'): ('MERGE', 'admin_1', 'sn',
'sn-th'),
('admin_1', 'sn', 'pout'): ('MERGE', 'admin_1', 'sn', 'sn-th'),
('admin_1', 'sn', 'nioro du rip'): ('MERGE', 'admin_1', 'sn',
'sn-kl'),
('admin_1', 'sn', 'sedhiou'): ('MERGE', 'admin_1', 'sn',
'sn-se'),
('admin_1', 'sn', 'diouloulou'): ('MERGE', 'admin_1', 'sn',
'sn-zg'),
('admin_1', 'sn', 'linguere'): ('MERGE', 'admin_1', 'sn',
'sn-lg'),
('admin_1', 'sn', 'kedougou'): ('MERGE', 'admin_1', 'sn',
'sn-ke'),
('admin_1', 'sn', 'thiadiaye'): ('MERGE', 'admin_1', 'sn',
'sn-th'),
('admin_1', 'sn', 'mekhe'): ('MERGE', 'admin_1', 'sn',
'sn-th'),
('admin_1', 'sn', 'koungheul'): ('MERGE', 'admin_1', 'sn',
'sn-ka'),
('admin_1', 'sn', 'coki'): ('MERGE', 'admin_1', 'sn', 'sn-lg'),
('admin_1', 'sn', 'poponguine'): ('MERGE', 'admin_1', 'sn',
'sn-th'),
('admin_1', 'sn', 'tivaoune'): ('MERGE', 'admin_1', 'sn',
'sn-th'),
('admin_1', 'sn', 'joal fadiouth'): ('MERGE', 'admin_1', 'sn',
'sn-th'),
('admin_1', 'sn', 'joal-fadiouth'): ('MERGE', 'admin_1', 'sn',
'sn-th'),
('admin_1', 'sn', 'bignona'): ('MERGE', 'admin_1', 'sn',
'sn-zg'),
('admin_1', 'sn', 'bikilane'): ('MERGE', 'admin_1', 'sn',
'sn-dk'), # Pikine??
('admin_1', 'sn', 'dioffior'): ('MERGE', 'admin_1', 'sn',
'sn-fk'),
('admin_1', 'sn', 'darou mousty'): ('MERGE', 'admin_1', 'sn',
'sn-lg'),
('admin_1', 'sn', 'thilogne'): ('MERGE', 'admin_1', 'sn',
'sn-mt'),
('admin_1', 'sn', 'bounkiling'): ('MERGE', 'admin_1', 'sn',
'sn-se'),
('admin_1', 'sn', 'khombole'): ('MERGE', 'admin_1', 'sn',
'sn-th'),
('admin_1', 'sn', 'richard-toll'): ('MERGE', 'admin_1', 'sn',
'sn-sl'),
('admin_1', 'sn', 'birkilane'): ('MERGE', 'admin_1', 'sn',
'sn-ka'),
('admin_1', 'sn', 'kebemer'): ('MERGE', 'admin_1', 'sn',
'sn-lg'),
('admin_1', 'sn', 'ranerou'): ('MERGE', 'admin_1', 'sn',
'sn-mt'),
('admin_1', 'sn', 'gossas'): ('MERGE', 'admin_1', 'sn',
'sn-fk'),
('admin_1', 'sn', 'bambey'): ('MERGE', 'admin_1', 'sn',
'sn-db'),
('admin_1', 'sn', 'sokone'): ('MERGE', 'admin_1', 'sn',
'sn-fk'),
('admin_1', 'sn', 'mékhé'): ('MERGE', 'admin_1', 'sn',
'sn-th'),
('admin_1', 'sn', 'joal'): ('MERGE', 'admin_1', 'sn', 'sn-th'),
('admin_1', 'sn', 'mbacké'): ('MERGE', 'admin_1', 'sn',
'sn-db'),
('admin_1', 'sn', 'vélingara'): ('MERGE', 'admin_1', 'sn',
'sn-kd'),
('admin_1', 'sn', 'bounkilig'): ('MERGE', 'admin_1', 'sn',
'sn-se'),
('admin_1', 'sn', 'diakhao'): ('MERGE', 'admin_1', 'sn',
'sn-fk'),
('admin_1', 'sn', 'linguère'): ('MERGE', 'admin_1', 'sn',
'sn-lg'),
('admin_1', 'sn', 'cocki'): ('MERGE', 'admin_1', 'sn',
'sn-lg'),
('admin_1', 'sn', 'kébémer'): ('MERGE', 'admin_1', 'sn',
'sn-lg'),
('admin_1', 'sn', 'nioro'): ('MERGE', 'admin_1', 'sn',
'sn-kl'),
('admin_1', 'sn', 'ranérou'): ('MERGE', 'admin_1', 'sn',
'sn-mt'),
('admin_1', 'sn', 'kanel'): ('MERGE', 'admin_1', 'sn',
'sn-mt'),
('admin_1', 'sn', 'joal fadiouth'): ('MERGE', 'admin_1', 'sn',
'sn-th'),
('admin_1', 'sn', 'passy'): ('MERGE', 'admin_1', 'sn',
'sn-fk'),
},
mode=MODE_STRICT)
self.update()
def get_datapoints(self):
r = []
for date in listdir(get_overseas_dir() / 'world_um' / 'api_data'):
print(date)
r.extend(self._get_datapoints(date))
return r
def _get_datapoints(self, date):
r = self.sdpf()
path = get_overseas_dir(
) / 'world_um' / 'api_data' / date / 'data.json'
print(path)
with open(path, 'r', encoding='utf-8') as f:
data = loads(f.read())
#print(data)
for country, country_data_dict in data['covid'].items():
for country_dict in country_data_dict['countries']:
key = 'smoothed_cli' if 'smoothed_cli' in country_dict else 'smoothed_covid_se'
if country_dict[key] is None:
continue
print(country_dict)
r.append(region_schema=Schemas.ADMIN_0,
region_parent='',
region_child=country,
datatype=DataTypes.FACEBOOK_COVID_SYMPTOMS,
value=round(country_dict[key] * 100000),
date_updated=self.convert_date(
country_dict['survey_date'],
formats=('%Y%m%d', )),
source_url=self.SOURCE_URL)
for region_dict in country_data_dict['regions']:
key = 'smoothed_cli' if 'smoothed_cli' in country_dict else 'smoothed_covid_se'
if region_dict[key] is None:
continue
print(region_dict)
r.append(region_schema=Schemas.ADMIN_1,
region_parent=country,
region_child=region_dict['region'],
datatype=DataTypes.FACEBOOK_COVID_SYMPTOMS,
value=round(region_dict[key] * 100000),
date_updated=self.convert_date(
region_dict['survey_date'],
formats=('%Y%m%d', )),
source_url=self.SOURCE_URL)
for country, country_data_dict in data['flu'].items():
for country_dict in country_data_dict['countries']:
key = 'smoothed_ili' if 'smoothed_ili' in country_dict else 'smoothed_flu_se'
if country_dict[key] is None:
continue
print(country_dict)
r.append(region_schema=Schemas.ADMIN_0,
region_parent='',
region_child=country,
datatype=DataTypes.FACEBOOK_FLU_SYMPTOMS,
value=round(country_dict[key] * 100000),
date_updated=self.convert_date(
country_dict['survey_date'],
formats=('%Y%m%d', )),
source_url=self.SOURCE_URL)
for region_dict in country_data_dict['regions']:
key = 'smoothed_ili' if 'smoothed_ili' in country_dict else 'smoothed_flu_se'
if region_dict[key] is None:
continue
print(region_dict)
r.append(region_schema=Schemas.ADMIN_1,
region_parent=country,
region_child=region_dict['region'],
datatype=DataTypes.FACEBOOK_FLU_SYMPTOMS,
value=round(region_dict[key] * 100000),
date_updated=self.convert_date(
region_dict['survey_date'],
formats=('%Y%m%d', )),
source_url=self.SOURCE_URL)
return r
def __init__(self):
# Only raw_data4.json is currently being updated,
# so won't download the others every day
GlobalBase.__init__(self, output_dir=get_overseas_dir() / 'rs' / 'data')
self.__today_dir = self.output_dir / datetime.datetime.now().strftime('%Y_%m_%d')
self.update()
def _get_datapoints(self):
# country_region_code,country_region,sub_region_1,sub_region_2,metro_area,iso_3166_2_code,census_fips_code,
# date,
# retail_and_recreation_percent_change_from_baseline,
# grocery_and_pharmacy_percent_change_from_baseline,
# parks_percent_change_from_baseline,
# transit_stations_percent_change_from_baseline,
# workplaces_percent_change_from_baseline,
# residential_percent_change_from_baseline
# AE,United Arab Emirates,,,,,,2020-02-15,0,4,5,0,2,1
# AE,United Arab Emirates,,,,,,2020-02-16,1,4,4,1,2,1
# AE,United Arab Emirates,,,,,,2020-02-17,-1,1,5,1,2,1
# AE,United Arab Emirates,,,,,,2020-02-18,-2,1,5,0,2,1
r = self.sdpf()
date = datetime.today().strftime('%Y_%m_%d')
path = get_overseas_dir(
) / 'world_google_mobility' / 'data' / date / 'mobility.csv'
with open(path, 'r', encoding='utf-8') as f:
for item in csv.DictReader(f):
date = self.convert_date(item['date'])
if item['iso_3166_2_code']:
region_schema = Schemas.ADMIN_1
region_parent = item['country_region_code']
region_child = item['iso_3166_2_code']
elif item['sub_region_2'] and item[
'country_region_code'] == 'AU':
region_schema = Schemas.LGA
region_parent = item['sub_region_1']
region_child = normalize_locality_name(
item['sub_region_2'])
elif not item['sub_region_1'] and not item['sub_region_2']:
region_schema = Schemas.ADMIN_0
region_parent = ''
region_child = item['country_region_code']
else:
continue
for datatype, value in (
(DataTypes.GOOGLE_MOBILITY_RETAIL_RECREATION,
item['retail_and_recreation_percent_change_from_baseline']
),
(DataTypes.GOOGLE_MOBILITY_SUPERMARKET_PHARMACY,
item['grocery_and_pharmacy_percent_change_from_baseline']
), (DataTypes.GOOGLE_MOBILITY_PARKS,
item['parks_percent_change_from_baseline']),
(DataTypes.GOOGLE_MOBILITY_PUBLIC_TRANSPORT,
item['transit_stations_percent_change_from_baseline']),
(DataTypes.GOOGLE_MOBILITY_WORKPLACES,
item['workplaces_percent_change_from_baseline']),
(DataTypes.GOOGLE_MOBILITY_RESIDENTIAL,
item['residential_percent_change_from_baseline'])):
if not value:
continue
r.append(region_schema=region_schema,
region_parent=region_parent,
region_child=region_child,
datatype=datatype,
value=int(value),
date_updated=date,
source_url=self.SOURCE_URL)
return r
