def __init__(self): URLBase.__init__(self, output_dir=get_overseas_dir() / 'lv' / 'data', urls_dict={ 'regions_data.json': URL(REGIONS_URL, static_file=False), }) self.sdpf = StrictDataPointsFactory(region_mappings={ ('admin_1', 'lv', 'daugavpils novads'): None, ('admin_1', 'lv', 'jelgavas novads'): None, ('admin_1', 'lv', 'jēkabpils novads'): None, ('admin_1', 'lv', 'kocēnu novads'): None, ('admin_1', 'lv', 'līgatnes novads'): None, ('admin_1', 'lv', 'pārgaujas novads'): None, ('admin_1', 'lv', 'pārgaujas novads'): None, ('admin_1', 'lv', 'priekuļu novads'): None, ('admin_1', 'lv', 'rēzeknes novads'): None, ('admin_1', 'lv', 'rūjienas novads'): None, ('admin_1', 'lv', 'strenču novads'): None, ('admin_1', 'lv', 'varakļānu novads'): None, ('admin_1', 'lv', 'ventspils novads'): None, ('admin_1', 'lv', 'viļānu novads'): None, }, mode=MODE_STRICT)
def __init__(self): URLBase.__init__( self, output_dir=get_overseas_dir() / 'sa' / 'data', urls_dict={ 'recovered_sum.json': URL('https://services6.arcgis.com/bKYAIlQgwHslVRaK/arcgis/rest/services/VWPlacesUniqueWithStatistics/FeatureServer/1/query?f=json&where=Recovered_SUM%3E0&returnGeometry=false&spatialRel=esriSpatialRelIntersects&outFields=*&orderByFields=Recovered_SUM%20desc&outSR=102100&resultOffset=0&resultRecordCount=1000&resultType=standard&cacheHint=true', static_file=False), 'tested_sum.json': URL('https://services6.arcgis.com/bKYAIlQgwHslVRaK/arcgis/rest/services/VWPlacesUniqueWithStatistics01/FeatureServer/1/query?f=json&where=Tested_SUM%3E0&returnGeometry=false&spatialRel=esriSpatialRelIntersects&outFields=*&orderByFields=Tested_SUM%20desc&outSR=102100&resultOffset=0&resultRecordCount=1000&resultType=standard&cacheHint=true', static_file=False), 'confirmed_sum.json': URL('https://services6.arcgis.com/bKYAIlQgwHslVRaK/arcgis/rest/services/VWPlacesUniqueWithStatistics/FeatureServer/1/query?f=json&where=1%3D1&returnGeometry=false&spatialRel=esriSpatialRelIntersects&outFields=*&orderByFields=Confirmed_SUM%20desc&outSR=102100&resultOffset=0&resultRecordCount=1000&resultType=standard&cacheHint=true', static_file=False) }) self.sdpf = StrictDataPointsFactory(region_mappings={ ('admin_1', 'sa', 'eastern region'): ('admin_1', 'sa', 'SA-04'), ('admin_1', 'sa', 'al qaseem'): ('admin_1', 'sa', 'SA-05'), ('admin_1', 'sa', 'aseer'): ('admin_1', 'sa', 'SA-14'), ('admin_1', 'sa', 'northern borders'): ('admin_1', 'sa', 'SA-08'), }, mode=MODE_STRICT) self.update()
def __init__(self): # Only raw_data4.json is currently being updated, # so won't download the others every day URLBase.__init__( self, output_dir=get_overseas_dir() / 'et' / 'data', urls_dict={ 'et_data.csv': URL( 'https://docs.google.com/spreadsheets/d/' '1wpMiswWu5En3Ljoqv-N9gK_VQWU6xYA9NBAYzJ3aE2g/' 'export?format=csv&' 'id=1wpMiswWu5En3Ljoqv-N9gK_VQWU6xYA9NBAYzJ3aE2g', static_file=False) }) self.sdpf = StrictDataPointsFactory(region_mappings={ ('admin_1', 'et', 'snnpr'): ('admin_1', 'et', 'et-sn'), ('admin_1', 'et', 'sidama'): None, ('admin_1', 'et', 'benshangul gumuz'): ('admin_1', 'et', 'et-be'), }, mode=MODE_STRICT) self.update()
def __init__(self): URLBase.__init__(self, output_dir=get_overseas_dir() / 'cz' / 'data', urls_dict={ 'national_tests.json': URL(TOTAL_TESTS_URL, static_file=False), 'national_confirmed.json': URL(TOTAL_CONFIRMED, static_file=False), 'national_stats.json': URL(TOTAL_NATIONAL_STATS, static_file=False), 'regional_cases.json': URL(INDIVIDUAL_CASES, static_file=False), 'regional_totals.json': URL(REGIONAL_TOTALS, static_file=False), 'regional_recovered.json': URL(RECOVERED, static_file=False), 'regional_deaths.json': URL(DEATHS, static_file=False), }) self.sdpf = StrictDataPointsFactory(region_mappings={ ('cz_okres', 'cz', 'cz010'): None, ('cz_okres', 'cz', 'cz020'): None, ('cz_okres', 'cz', 'cz031'): None, ('cz_okres', 'cz', 'cz032'): None, ('cz_okres', 'cz', 'cz041'): None, ('cz_okres', 'cz', 'cz042'): None, ('cz_okres', 'cz', 'cz051'): None, ('cz_okres', 'cz', 'cz052'): None, ('cz_okres', 'cz', 'cz053'): None, ('cz_okres', 'cz', 'cz064'): None, ('cz_okres', 'cz', 'cz071'): None, ('cz_okres', 'cz', 'cz063'): None, ('cz_okres', 'cz', 'cz072'): None, ('cz_okres', 'cz', 'cz080'): None, }, mode=MODE_STRICT) self.update()
def __init__(self): # Only raw_data4.json is currently being updated, # so won't download the others every day URLBase.__init__(self, output_dir=get_overseas_dir() / 'hk' / 'data', urls_dict={ 'T2_Data.json': URL( 'https://chp-dashboard.geodata.gov.hk/covid-19/T2_Data.json', static_file=False ), 'cases.json': URL( """https://services8.arcgis.com/PXQv9PaDJHzt8rp0/arcgis/rest/services/Merge_Display_0227_View/FeatureServer/0/query?f=json&where=(Status = 'Existing' OR Status = 'History') AND (Case_no_ <> -1) AND (Status = 'History')&returnGeometry=true&spatialRel=esriSpatialRelIntersects&maxAllowableOffset=38&geometry={"xmin":12699553.627367351,"ymin":2543824.3013545386,"xmax":12719121.506608326,"ymax":2563392.180595517,"spatialReference":{"wkid":102100}}&geometryType=esriGeometryEnvelope&inSR=102100&outFields=*&outSR=102100&resultType=tile""".replace(' ', '%20'), static_file=False ) } ) self.sdpf = StrictDataPointsFactory( region_mappings={ ('hk_district', 'hk', '黄大仙'): ('hk_district', 'hk', '黃大仙'), ('hk_district', 'hk', '荃灣區'): ('hk_district', 'hk', '荃灣'), }, mode=MODE_STRICT ) self.update()
def __init__(self): URLBase.__init__(self, output_dir=get_overseas_dir() / 'my' / 'esri_dash_data', urls_dict={ 'district_case.json': URL( DISTRICT_CASE, static_file=False ), 'sub_district_case.json': URL( SUB_DISTRICT_CASE, static_file=False ), } ) self.sdpf = StrictDataPointsFactory( region_mappings={ ('admin_1', 'my', 'johor'): ('admin_1', 'my', 'my-01'), ('admin_1', 'my', 'kedah'): ('admin_1', 'my', 'my-02'), ('admin_1', 'my', 'kelantan'): ('admin_1', 'my', 'my-03'), ('admin_1', 'my', 'kuala lumpur'): ('admin_1', 'my', 'my-14'), ('admin_1', 'my', 'labuan'): ('admin_1', 'my', 'my-15'), ('admin_1', 'my', 'melaka'): ('admin_1', 'my', 'my-04'), ('admin_1', 'my', 'my-01'): ('admin_1', 'my', 'my-01'), ('admin_1', 'my', 'my-02'): ('admin_1', 'my', 'my-02'), ('admin_1', 'my', 'my-03'): ('admin_1', 'my', 'my-03'), ('admin_1', 'my', 'my-04'): ('admin_1', 'my', 'my-04'), ('admin_1', 'my', 'my-05'): ('admin_1', 'my', 'my-05'), ('admin_1', 'my', 'my-06'): ('admin_1', 'my', 'my-06'), ('admin_1', 'my', 'my-07'): ('admin_1', 'my', 'my-07'), ('admin_1', 'my', 'my-08'): ('admin_1', 'my', 'my-08'), ('admin_1', 'my', 'my-09'): ('admin_1', 'my', 'my-09'), ('admin_1', 'my', 'my-10'): ('admin_1', 'my', 'my-10'), ('admin_1', 'my', 'my-11'): ('admin_1', 'my', 'my-11'), ('admin_1', 'my', 'my-12'): ('admin_1', 'my', 'my-12'), ('admin_1', 'my', 'my-13'): ('admin_1', 'my', 'my-13'), ('admin_1', 'my', 'my-14'): ('admin_1', 'my', 'my-14'), ('admin_1', 'my', 'my-15'): ('admin_1', 'my', 'my-15'), ('admin_1', 'my', 'my-16'): ('admin_1', 'my', 'my-16'), ('admin_1', 'my', 'negeri sembilan'): ('admin_1', 'my', 'my-05'), ('admin_1', 'my', 'pahang'): ('admin_1', 'my', 'my-06'), ('admin_1', 'my', 'perak'): ('admin_1', 'my', 'my-08'), ('admin_1', 'my', 'perlis'): ('admin_1', 'my', 'my-09'), ('admin_1', 'my', 'pulau pinang'): ('admin_1', 'my', 'my-07'), ('admin_1', 'my', 'putrajaya'): ('admin_1', 'my', 'my-16'), ('admin_1', 'my', 'sabah'): ('admin_1', 'my', 'my-12'), ('admin_1', 'my', 'sarawak'): ('admin_1', 'my', 'my-13'), ('admin_1', 'my', 'selangor'): ('admin_1', 'my', 'my-10'), ('admin_1', 'my', 'terengganu'): ('admin_1', 'my', 'my-11'), ('admin_1', 'my', 'tiada data'): None, ('my_district', 'my', 'alor gajah'): ('my_district', 'my', 'alor gajah'), ('my_district', 'my', 'asajaya'): ('my_district', 'my', 'asajaya'), ('my_district', 'my', 'bachok'): ('my_district', 'my', 'bachok'), ('my_district', 'my', 'baling'): ('my_district', 'my', 'baling'), ('my_district', 'my', 'bandar baharu'): ('my_district', 'my', 'bandar baharu'), ('my_district', 'my', 'bandar bahru'): ('my_district', 'my', 'bandar baharu'), ('my_district', 'my', 'bandar tun razak'): None, ('my_district', 'my', 'bangsar'): None, ('my_district', 'my', 'barat daya'): ('my_district', 'my', 'barat daya'), ('my_district', 'my', 'batang padang'): ('my_district', 'my', 'batang padang'), ('my_district', 'my', 'batu'): None, ('my_district', 'my', 'batu pahat'): ('my_district', 'my', 'batu pahat'), ('my_district', 'my', 'bau'): ('my_district', 'my', 'bau'), ('my_district', 'my', 'beaufort'): ('my_district', 'my', 'beaufort'), ('my_district', 'my', 'bentong'): ('my_district', 'my', 'bentong'), ('my_district', 'my', 'bera'): ('my_district', 'my', 'bera'), ('my_district', 'my', 'besut'): ('my_district', 'my', 'besut'), ('my_district', 'my', 'betong'): ('my_district', 'my', 'betong'), ('my_district', 'my', 'bintulu'): ('my_district', 'my', 'bintulu'), ('my_district', 'my', 'brickfields'): None, ('my_district', 'my', 'bukit damansara'): None, ('my_district', 'my', 'bukit jalil'): None, ('my_district', 'my', 'bukit bintang'): None, ('my_district', 'my', 'cameron highlands'): ('my_district', 'my', 'cameron highlands'), ('my_district', 'my', 'cheras'): None, ('my_district', 'my', 'dalat'): ('my_district', 'my', 'dalat'), ('my_district', 'my', 'daro'): ('my_district', 'my', 'daro'), ('my_district', 'my', 'dungun'): ('my_district', 'my', 'dungun'), ('my_district', 'my', 'duta'): None, ('my_district', 'my', 'gombak'): ('my_district', 'my', 'gombak'), ('my_district', 'my', 'gua musang'): ('my_district', 'my', 'gua musang'), ('my_district', 'my', 'hilir perak'): ('my_district', 'my', 'hilir perak'), ('my_district', 'my', 'hulu langat'): None, ('my_district', 'my', 'hulu perak'): None, ('my_district', 'my', 'hulu selangor'): None, ('my_district', 'my', 'hulu terengganu'): ('my_district', 'my', 'hulu terengganu'), ('my_district', 'my', 'ibu kota'): None, ('my_district', 'my', 'jasin'): ('my_district', 'my', 'jasin'), ('my_district', 'my', 'jelebu'): ('my_district', 'my', 'jelebu'), ('my_district', 'my', 'jeli'): ('my_district', 'my', 'jeli'), ('my_district', 'my', 'jempol'): ('my_district', 'my', 'jempol'), ('my_district', 'my', 'jerantut'): ('my_district', 'my', 'jerantut'), ('my_district', 'my', 'johor bahru'): ('my_district', 'my', 'johor bahru'), ('my_district', 'my', 'julau'): ('my_district', 'my', 'julau'), ('my_district', 'my', 'kampar'): ('my_district', 'my', 'kampar'), ('my_district', 'my', 'kampung bharu'): None, ('my_district', 'my', 'kangar'): None, ('my_district', 'my', 'kanowit'): ('my_district', 'my', 'kanowit'), ('my_district', 'my', 'kapit'): ('my_district', 'my', 'kapit'), ('my_district', 'my', 'kemaman'): ('my_district', 'my', 'kemaman'), ('my_district', 'my', 'keningau'): ('my_district', 'my', 'keningau'), ('my_district', 'my', 'kepong'): None, ('my_district', 'my', 'kerian'): ('my_district', 'my', 'kerian'), ('my_district', 'my', 'kinabatangan'): ('my_district', 'my', 'kinabatangan'), ('my_district', 'my', 'kinta'): ('my_district', 'my', 'kinta'), ('my_district', 'my', 'klang'): ('my_district', 'my', 'klang'), ('my_district', 'my', 'kluang'): ('my_district', 'my', 'kluang'), ('my_district', 'my', 'kota belud'): ('my_district', 'my', 'kota belud'), ('my_district', 'my', 'kota bharu'): ('my_district', 'my', 'kota bharu'), ('my_district', 'my', 'kota kinabalu'): ('my_district', 'my', 'kota kinabalu'), ('my_district', 'my', 'kota marudu'): ('my_district', 'my', 'kota marudu'), ('my_district', 'my', 'kota samarahan'): None, ('my_district', 'my', 'kota setar'): ('my_district', 'my', 'kota setar'), ('my_district', 'my', 'kota tinggi'): ('my_district', 'my', 'kota tinggi'), ('my_district', 'my', 'kuala kangsar'): ('my_district', 'my', 'kuala kangsar'), ('my_district', 'my', 'kuala krai'): ('my_district', 'my', 'kuala krai'), ('my_district', 'my', 'kuala langat'): ('my_district', 'my', 'kuala langat'), ('my_district', 'my', 'kuala lumpur'): None, # FIXME!!!!! =================================================== ('my_district', 'my', 'setiawangsa'): None, ('my_district', 'my', 'kuala muda'): ('my_district', 'my', 'kuala muda'), ('my_district', 'my', 'kuala pilah'): ('my_district', 'my', 'kuala pilah'), ('my_district', 'my', 'kuala selangor'): ('my_district', 'my', 'kuala selangor'), ('my_district', 'my', 'kuala terengganu'): ('my_district', 'my', 'kuala terengganu'), ('my_district', 'my', 'kuantan'): ('my_district', 'my', 'kuantan'), ('my_district', 'my', 'kubang pasu'): ('my_district', 'my', 'kubang pasu'), ('my_district', 'my', 'kuching'): ('my_district', 'my', 'kuching'), ('my_district', 'my', 'kudat'): ('my_district', 'my', 'kudat'), ('my_district', 'my', 'kulai'): None, ('my_district', 'my', 'kulim'): ('my_district', 'my', 'kulim'), ('my_district', 'my', 'labuan'): None, ('my_district', 'my', 'lahad datu'): ('my_district', 'my', 'lahad datu'), ('my_district', 'my', 'langkawi'): ('my_district', 'my', 'langkawi'), ('my_district', 'my', 'larut, matang, selama'): None, ('my_district', 'my', 'lawas'): ('my_district', 'my', 'lawas'), ('my_district', 'my', 'lembah pantai'): None, ('my_district', 'my', 'limbang'): ('my_district', 'my', 'limbang'), ('my_district', 'my', 'lipis'): ('my_district', 'my', 'lipis'), ('my_district', 'my', 'lubok antu'): ('my_district', 'my', 'lubok antu'), ('my_district', 'my', 'machang'): ('my_district', 'my', 'machang'), ('my_district', 'my', 'manjalara'): None, ('my_district', 'my', 'manjung'): None, ('my_district', 'my', 'maran'): ('my_district', 'my', 'maran'), ('my_district', 'my', 'marang'): ('my_district', 'my', 'marang'), ('my_district', 'my', 'matu'): ('my_district', 'my', 'matu'), ('my_district', 'my', 'melaka tengah'): ('my_district', 'my', 'melaka tengah'), ('my_district', 'my', 'mersing'): ('my_district', 'my', 'mersing'), ('my_district', 'my', 'miri'): ('my_district', 'my', 'miri'), ('my_district', 'my', 'mont kiara'): None, ('my_district', 'my', 'muar'): ('my_district', 'my', 'muar'), ('my_district', 'my', 'mukah'): ('my_district', 'my', 'mukah'), ('my_district', 'my', 'nabawan'): ('my_district', 'my', 'nabawan'), ('my_district', 'my', 'padang terap'): ('my_district', 'my', 'padang terap'), ('my_district', 'my', 'pakan'): ('my_district', 'my', 'pakan'), ('my_district', 'my', 'pandan'): None, ('my_district', 'my', 'pantai barat utara'): None, ('my_district', 'my', 'papar'): ('my_district', 'my', 'papar'), ('my_district', 'my', 'pasir mas'): ('my_district', 'my', 'pasir mas'), ('my_district', 'my', 'pasir puteh'): ('my_district', 'my', 'pasir puteh'), ('my_district', 'my', 'pekan'): ('my_district', 'my', 'pekan'), ('my_district', 'my', 'penampang'): ('my_district', 'my', 'penampang'), ('my_district', 'my', 'pendang'): ('my_district', 'my', 'pendang'), ('my_district', 'my', 'perak tengah'): ('my_district', 'my', 'perak tengah'), ('my_district', 'my', 'petaling'): ('my_district', 'my', 'petaling'), ('my_district', 'my', 'pitas'): ('my_district', 'my', 'pitas'), ('my_district', 'my', 'pokok sena'): ('my_district', 'my', 'pokok sena'), ('my_district', 'my', 'pontian'): ('my_district', 'my', 'pontian'), ('my_district', 'my', 'port dickson'): ('my_district', 'my', 'port dickson'), ('my_district', 'my', 'pudu'): None, ('my_district', 'my', 'putatan'): ('my_district', 'my', 'putatan'), ('my_district', 'my', 'putrajaya'): None, ('my_district', 'my', 'ranau'): ('my_district', 'my', 'ranau'), ('my_district', 'my', 'raub'): ('my_district', 'my', 'raub'), ('my_district', 'my', 'rembau'): ('my_district', 'my', 'rembau'), ('my_district', 'my', 'rompin'): ('my_district', 'my', 'rompin'), ('my_district', 'my', 'sabak bernam'): ('my_district', 'my', 'sabak bernam'), ('my_district', 'my', 'samarahan'): ('my_district', 'my', 'samarahan'), ('my_district', 'my', 'sandakan'): ('my_district', 'my', 'sandakan'), ('my_district', 'my', 'saratok'): ('my_district', 'my', 'saratok'), ('my_district', 'my', 'sarikei'): ('my_district', 'my', 'sarikei'), ('my_district', 'my', 'seberang perai selatan'): None, ('my_district', 'my', 'seberang perai tengah'): None, ('my_district', 'my', 'seberang perai utara'): None, ('my_district', 'my', 'segamat'): ('my_district', 'my', 'segamat'), ('my_district', 'my', 'semporna'): ('my_district', 'my', 'semporna'), ('my_district', 'my', 'sentul'): None, ('my_district', 'my', 'sepang'): ('my_district', 'my', 'sepang'), ('my_district', 'my', 'seputeh'): None, ('my_district', 'my', 'seremban'): ('my_district', 'my', 'seremban'), ('my_district', 'my', 'serian'): ('my_district', 'my', 'serian'), ('my_district', 'my', 'setiu'): ('my_district', 'my', 'setiu'), ('my_district', 'my', 'sibu'): ('my_district', 'my', 'sibu'), ('my_district', 'my', 'sik'): ('my_district', 'my', 'sik'), ('my_district', 'my', 'simunjan'): ('my_district', 'my', 'simunjan'), ('my_district', 'my', 'sipitang'): ('my_district', 'my', 'sipitang'), ('my_district', 'my', 'song'): ('my_district', 'my', 'song'), ('my_district', 'my', 'sri aman'): ('my_district', 'my', 'sri aman'), ('my_district', 'my', 'sri petaling'): None, ('my_district', 'my', 'tambunan'): ('my_district', 'my', 'tambunan'), ('my_district', 'my', 'tampin'): ('my_district', 'my', 'tampin'), ('my_district', 'my', 'tanah merah'): ('my_district', 'my', 'tanah merah'), ('my_district', 'my', 'tangkak'): None, ('my_district', 'my', 'tatau'): ('my_district', 'my', 'tatau'), ('my_district', 'my', 'tawau'): ('my_district', 'my', 'tawau'), ('my_district', 'my', 'temerloh'): ('my_district', 'my', 'temerloh'), ('my_district', 'my', 'tenom'): ('my_district', 'my', 'tenom'), ('my_district', 'my', 'timur laut'): ('my_district', 'my', 'timur laut'), ('my_district', 'my', 'titiwangsa'): None, ('my_district', 'my', 'tuaran'): ('my_district', 'my', 'tuaran'), ('my_district', 'my', 'tumpat'): ('my_district', 'my', 'tumpat'), ('my_district', 'my', 'wangsa maju'): None, ('my_district', 'my', 'yan'): ('my_district', 'my', 'yan'), ('my_district', 'my', 'kuala nerus'): None, ('my_district', 'my', 'telupid'): None, ('my_district', 'my', 'segambut'): None, }, mode=MODE_STRICT ) self.update()
def __init__(self): GithubRepo.__init__(self, output_dir=get_overseas_dir() / 'lk' / 'covid19-srilankan-data', github_url='https://github.com/arimacdev/covid19-srilankan-data') self.sdpf = StrictDataPointsFactory(mode=MODE_STRICT) self.update()
def __init__(self): GithubRepo.__init__( self, output_dir=get_overseas_dir() / 'us_nytimes' / 'covid-19-data', github_url='https://github.com/nytimes/covid-19-data') self.sdpf = StrictDataPointsFactory(region_mappings={ ('us_county', 'alaska', 'fairbanks north star borough'): None, ('us_county', 'alaska', 'ketchikan gateway borough'): None, ('us_county', 'alaska', 'kenai peninsula borough'): None, ('us_county', 'missouri', 'kansas city'): None, ('us_county', 'alaska', 'juneau city and borough'): None, ('us_county', 'alaska', 'matanuska-susitna borough'): None, ('us_county', 'alaska', 'yukon-koyukuk census area'): None, ('us_county', 'alaska', 'southeast fairbanks census area'): None, ('us_county', 'alaska', 'petersburg borough'): None, ('us_county', 'alaska', 'bethel census area'): None, ('us_county', 'alaska', 'prince of wales-hyder census area'): None, ('us_county', 'alaska', 'nome census area'): None, ('us_county', 'alaska', 'kodiak island borough'): None, ('us_county', 'alaska', 'sitka city and borough'): None, ('us_county', 'puerto rico', 'anasco'): None, ('us_county', 'puerto rico', 'bayamon'): None, ('us_county', 'puerto rico', 'canovanas'): None, ('us_county', 'puerto rico', 'catano'): None, ('us_county', 'puerto rico', 'comerio'): None, ('us_county', 'puerto rico', 'guanica'): None, ('us_county', 'puerto rico', 'juana diaz'): None, ('us_county', 'puerto rico', 'las marias'): None, ('us_county', 'puerto rico', 'loiza'): None, ('us_county', 'puerto rico', 'manati'): None, ('us_county', 'puerto rico', 'mayaguez'): None, ('us_county', 'puerto rico', 'penuelas'): None, ('us_county', 'puerto rico', 'rincon'): None, ('us_county', 'puerto rico', 'rio grande'): None, ('us_county', 'puerto rico', 'san german'): None, ('us_county', 'puerto rico', 'san sebastian'): None, ('us_county', 'alaska', 'valdez-cordova census area'): None, ('us_county', 'alaska', 'northwest arctic borough'): None, ('us_county', 'alaska', 'north slope borough'): None, ('us_county', 'alaska', 'bristol bay borough'): None, ('us_county', 'alaska', 'dillingham census area'): None, ('us_county', 'alaska', 'aleutians west census area'): None, ('us_county', 'alaska', 'lake and peninsula borough'): None, ('us_county', 'alaska', 'wrangell city and borough'): None, ('us_county', 'alaska', 'aleutians east borough'): None, ('us_county', 'alaska', 'haines borough'): None, ('us_county', 'alaska', 'denali borough'): None, ('us_county', 'alaska', 'kusilvak census area'): None, ('us_county', 'alaska', 'skagway municipality'): None, ('us_county', 'missouri', 'joplin'): None, ('admin_1', 'us', 'puerto rico'): None, ('admin_1', 'us', 'virgin islands'): None, ('admin_1', 'us', 'guam'): None, ('admin_1', 'us', 'northern mariana islands'): None, }, mode=MODE_STRICT) self.update()
def _get_positive_by_department(self, date, totals, added_totals, tests, added_tests): out = DataPointMerger() base_path = get_overseas_dir() / 'fr' / 'esridata' / date for fnam in sorted(listdir(base_path)): r = self.sdpf() with open(base_path / fnam, 'r', encoding='utf-8') as f: data = json.loads(f.read()) print(base_path, fnam, data) for property in data['features']: attributes = property['attributes'] print(property) date = self.convert_date(attributes['Jour']) try: region_child = 'FR-%02d' % int(attributes['CODE_DEPT']) except ValueError: # e.g. 2A region_child = 'FR-%s' % attributes['CODE_DEPT'] for datatype, value in ( (DataTypes.STATUS_HOSPITALIZED, attributes['Hospitalisation_T']), #(DataTypes.FIXME, attributes['Hospitalisation_H']), #(DataTypes.FIXME, attributes['Hospitalisation_F']), (DataTypes.STATUS_ICU, attributes['Reanimation_T']), #(DataTypes.FIXME, attributes['Reanimation_H']), #(DataTypes.FIXME, attributes['Reanimation_F']), (DataTypes.STATUS_DEATHS, attributes['Deces_T']), #(DataTypes.FIXME, attributes['Deces_H']), #(DataTypes.FIXME, attributes['Deces_F']), (DataTypes.NEW, attributes['Tests_Viro_P']), #(DataTypes.TESTS_TOTAL, attributes['Tests_Viro_T']) # FIXME: This is new tests! ): if value is None: continue r.append( region_schema=Schemas.ADMIN_1, region_parent='FR', region_child=region_child, datatype=datatype, value=value, date_updated=date, source_url=self.SOURCE_URL ) # I don't think Nbre_Cas_Confirmes is ever not None assert attributes['Nbre_Cas_Confirmes'] is None, attributes if attributes['Tests_Viro_P'] and not date in added_totals[region_child]: added_totals[region_child].add(date) totals[region_child] += attributes['Tests_Viro_P']# or attributes['Nbre_Cas_Confirmes']) r.append( region_schema=Schemas.ADMIN_1, region_parent='FR', region_child=region_child, datatype=DataTypes.TOTAL, value=totals[region_child], date_updated=date, source_url=self.SOURCE_URL ) if attributes['Tests_Viro_T'] and not date in added_tests[region_child]: added_tests[region_child].add(date) tests[region_child] += attributes['Tests_Viro_T'] r.append( region_schema=Schemas.ADMIN_1, region_parent='FR', region_child=region_child, datatype=DataTypes.TESTS_TOTAL, value=tests[region_child], date_updated=date, source_url=self.SOURCE_URL ) out.extend(r) return out
def __init__(self): # Only raw_data4.json is currently being updated, # so won't download the others every day URLBase.__init__( self, output_dir=get_overseas_dir() / 'co' / 'data', urls_dict={ 'co_data.csv': URL('https://proxy.hxlstandard.org/data.csv?dest=data_edit&tagger-match-all=on&tagger-01-header=id+de+caso&tagger-01-tag=%23meta+%2Bid&tagger-02-header=fecha+de+notificacion&tagger-02-tag=%23date+%2Bnotification&tagger-03-header=codigo+divipola&tagger-03-tag=%23code&tagger-04-header=ciudad+de+ubicacion&tagger-04-tag=%23adm3+%2Bname&tagger-05-header=departamento+o+distrito&tagger-05-tag=%23adm2+%2Bname&tagger-06-header=atencion&tagger-06-tag=%23indicator+%2Binfected+%2Btype&tagger-07-header=edad&tagger-07-tag=%23indicator+%2Binfected+%2Bage&tagger-08-header=sexo&tagger-08-tag=%23indicator+%2Binfected+%2Bsex&tagger-10-header=estado&tagger-10-tag=%23indicator+%2Binfected+%2Bstatus&tagger-11-header=pais+de+procedencia&tagger-11-tag=%23indicator+%2Binfected+%2Borigin&tagger-13-header=fecha+de+muerte&tagger-13-tag=%23date+%2Breported+%2Bdeath&tagger-14-header=fecha+diagnostico&tagger-14-tag=%23date+%2Breported+%2Bnotification&tagger-15-header=fecha+recuperado&tagger-15-tag=%23date+%2Breported+%2Brecovered&tagger-16-header=fecha+reporte+web&tagger-16-tag=%23date+%2Breported&header-row=1&url=https%3A%2F%2Fwww.datos.gov.co%2Fapi%2Fviews%2Fgt2j-8ykr%2Frows.csv%3FaccessType%3DDOWNLOAD', static_file=False), }) self.sdpf = StrictDataPointsFactory( region_mappings={ ('admin_1', 'co', 'co-dc'): ('MERGE', 'admin_1', 'co', 'co-cun'), ('admin_1', 'co', 'co-cun'): ('MERGE', 'admin_1', 'co', 'co-cun'), ('admin_1', 'co', 'cartagena d.t. y c.'): ('MERGE', 'admin_1', 'co', 'co-bol'), ('admin_1', 'co', 'cartagena'): ('MERGE', 'admin_1', 'co', 'co-bol'), ('admin_1', 'co', 'barranquilla d.e.'): ('MERGE', 'admin_1', 'co', 'co-atl'), ('admin_1', 'co', 'barranquilla'): ('MERGE', 'admin_1', 'co', 'co-atl'), ('admin_1', 'co', 'santa marta d.t. y c.'): ('MERGE', 'admin_1', 'co', 'co-atl'), ('admin_1', 'co', 'santa marta'): ('MERGE', 'admin_1', 'co', 'co-atl'), ('admin_1', 'co', 'archipiélago de san andrés providencia y santa catalina'): ('MERGE', 'admin_1', 'co', 'co-atl'), ('admin_1', 'co', 'buenaventura d.e.'): ('MERGE', 'admin_1', 'co', 'co-vac'), ('admin_1', 'co', 'buenaventura'): ('MERGE', 'admin_1', 'co', 'co-vac'), ('admin_1', 'co', 'norte santander'): ('MERGE', 'admin_1', 'co', 'co-vac'), ('co_municipality', 'cartagena d.t. y c.', 'cartagena de indias'): ('co_municipality', 'co-bol', 'cartagena de indias'), ('co_municipality', 'norte de santander', 'cúcuta'): None, ('co_municipality', 'barranquilla d.e.', 'barranquilla'): ('co_municipality', 'co-atl', 'barranquilla'), ('co_municipality', 'santa marta d.t. y c.', 'santa marta'): None, ('co_municipality', 'archipiélago de san andrés providencia y santa catalina', 'san andrés'): None, ('co_municipality', 'archipiélago de san andrés providencia y santa catalina', 'providencia'): None, ('co_municipality', 'buenaventura d.e.', 'buenaventura'): ('co_municipality', 'co-vac', 'buenaventura'), ('co_municipality', 'co-dc', 'bogotá d.c.'): None, # FIXME!!!! ('co_municipality', 'cundinamarca', 'villa de de ubaté'): None, ('co_municipality', 'nariño', 'cuaspúd'): None, ('co_municipality', 'nariño', 'ancuyá'): None, ('co_municipality', 'cauca', 'piendamó'): None, ('co_municipality', 'cauca', 'guapí'): None, ('co_municipality', 'cauca', 'sotara'): None, ('co_municipality', 'vaupés', 'mitú'): None, ('co_municipality', 'vaupés', 'yavaraté'): None, ('co_municipality', 'vaupés', 'taraira'): None, ('co_municipality', 'chocó', 'el cantón san pablo'): None, ('co_municipality', 'chocó', 'el litoral san juan'): None, ('co_municipality', 'sucre', 'tolú viejo'): None, ('co_municipality', 'sucre', 'coloso'): None, ('co_municipality', 'antioquia', 'san pedro los milagros'): None, ('co_municipality', 'antioquia', 'san josé la montaña'): None, ('co_municipality', 'meta', 'san luis de cubarral'): None, ('co_municipality', 'santander', 'el carmen de chucurí'): None, ('co_municipality', 'vaupés', 'carurú'): None, ('co_municipality', 'guainía', 'barranco minas'): None, }, mode=MODE_STRICT) self.update()
def __init__(self): GithubRepo.__init__(self, output_dir=get_overseas_dir() / 'nz' / 'nz-covid19-data', github_url='https://github.com/philiprenich/nz-covid19-data') self.sdpf = StrictDataPointsFactory() self.update()
def get_datapoints(self): r = [] dpm = DataPointMerger() for date in sorted(listdir(get_overseas_dir() / 'uk' / 'gov-api')): r.extend(self._get_datapoints(date, dpm)) return r
def __init__(self): # Only raw_data4.json is currently being updated, # so won't download the others every day URLBase.__init__( self, output_dir=get_overseas_dir() / 'ly' / 'data', urls_dict={ 'ly_data.csv': URL( 'https://docs.google.com/spreadsheets/d/e/' '2PACX-1vQQWJZmGZJfUm22CPWoeW6rSS7Xh4K54r4A8RlN214ZCIPBUBOug3UbxFPrbiT3FQic6HS8wGdUhv3f/' 'pub?output=csv', static_file=False) }) self.sdpf = StrictDataPointsFactory( region_mappings={ ('admin_1', 'ly', 'اجدابيا'): None, # Ajdabiya, Al Wahat ('admin_1', 'ly', 'البريقة'): None, # Brega, Al Wahat ('admin_1', 'ly', 'البيضاء'): None, # Bayda, Jabal al Akhdar ('admin_1', 'ly', 'الجفارة'): None, # Jafara ('admin_1', 'ly', 'الجميل'): None, # Aljmail, Nuqat al Khams ('admin_1', 'ly', 'الخمس'): None, # Al-Khums, Murqub ('admin_1', 'ly', 'الرجبان'): None, # Alrujban, ??? ('admin_1', 'ly', 'الرحيبات'): None, # Alruhaibat, ??? ('admin_1', 'ly', 'الرياينة'): None, # Alriyayna, ??? ('admin_1', 'ly', 'الزاوية'): None, # Az-Zāwiyah, ??? ('admin_1', 'ly', 'الزنتان'): None, # Alzintan, Jabal al Gharbi ('admin_1', 'ly', 'العجيلات'): None, # Ajaylat, Nuqat al Khams ('admin_1', 'ly', 'العزيزية'): None, # ʽAziziya, Jafara ('admin_1', 'ly', 'القلعة'): None, ('admin_1', 'ly', 'الكفرة'): None, # Kufra District ('admin_1', 'ly', 'المحروقة'): None, ('admin_1', 'ly', 'المرج'): None, # Marj, Marj ('admin_1', 'ly', 'بني وليد'): None, # Bani Walid, Misrata District ('admin_1', 'ly', 'جادو'): None, ('admin_1', 'ly', 'جنزور'): None, # Janzur, Janzour/Greater Tripoli??? ('admin_1', 'ly', 'درنة'): None, # Derna, Derna ('admin_1', 'ly', 'رقدالين'): None, ('admin_1', 'ly', 'زلطن'): None, ('admin_1', 'ly', 'زليتن'): None, ('admin_1', 'ly', 'زوارة'): None, ('admin_1', 'ly', 'سبها'): None, ('admin_1', 'ly', 'سرت'): None, ('admin_1', 'ly', 'صبراتة'): None, ('admin_1', 'ly', 'صرمان'): None, ('admin_1', 'ly', 'طبرق'): None, ('admin_1', 'ly', 'طرابلس'): None, ('admin_1', 'ly', 'غدامس'): None, ('admin_1', 'ly', 'غريان'): None, ('admin_1', 'ly', 'قصر الاخيار'): None, ('admin_1', 'ly', 'قصر بن غشير'): None, ('admin_1', 'ly', 'مزدة'): None, ('admin_1', 'ly', 'مسلاتة'): None, ('admin_1', 'ly', 'مصراتة'): None, ('admin_1', 'ly', 'نالوت'): None, ('admin_1', 'ly', 'هراوة'): None, ('admin_1', 'ly', 'وادي الشاطيء'): None, ('admin_1', 'ly', 'يفرن'): None, ('admin_1', 'ly', 'سوسة'): None, ('admin_1', 'ly', 'تيجي'): None, ('admin_1', 'ly', 'امساعد'): None, ('admin_1', 'ly', 'الزهرة'): None, ('admin_1', 'ly', 'السبيعة'): None, ('admin_1', 'ly', 'تندميرة'): None, ('admin_1', 'ly', 'بن جواد'): None, ('admin_1', 'ly', 'الأبيار'): None, ('admin_1', 'ly', 'كاباو'): None, ('admin_1', 'ly', 'الجفرة'): None, ('admin_1', 'ly', 'ككلة'): None, ('admin_1', 'ly', 'الشويرف'): None, ('admin_1', 'ly', 'ترهونة'): None, }, mode=MODE_STRICT) self.update()
def __download_dataset(self, area_type): """ Extracts paginated data by requesting all of the pages and combining the results. """ date = datetime.datetime.now().strftime('%Y_%m_%d') dir_ = get_overseas_dir() / 'uk' / 'gov-api' / date if not exists(dir_): makedirs(dir_) path = dir_ / f'{area_type}.json' if exists(path): # Don't download if already downloaded! return endpoint = "https://api.coronavirus.data.gov.uk/v1/data" structure = { "date": "date", "name": "areaName", "code": "areaCode", "daily": "newCasesBySpecimenDate", "cumulative": "cumCasesBySpecimenDate", "deathsDaily": "newDeaths28DaysByPublishDate", "deathsCumulative": "cumDeaths28DaysByPublishDate", "cumTestsByPublishDate": "cumTestsByPublishDate", "newTestsByPublishDate": "newTestsByPublishDate", "newAdmissions": "newAdmissions", "cumAdmissions": "cumAdmissions", "cumAdmissionsByAge": "cumAdmissionsByAge", "hospitalCases": "hospitalCases", "covidOccupiedMVBeds": "covidOccupiedMVBeds", "maleCases": "maleCases", "femaleCases": "femaleCases" } api_params = { "filters": f"areaType={area_type}", "structure": dumps(structure, separators=(",", ":")), "format": "json" } data = [] page_number = 1 while True: # Adding page number to query params api_params["page"] = page_number print("getting:", endpoint, api_params) response = get(endpoint, params=api_params, timeout=30) if response.status_code >= HTTPStatus.BAD_REQUEST: raise RuntimeError(f'Request failed: {response.text}') elif response.status_code == HTTPStatus.NO_CONTENT: break current_data = response.json() page_data = current_data['data'] data.extend(page_data) #print(page_data) # The "next" attribute in "pagination" will be `None` # when we reach the end. if current_data["pagination"]["next"] is None: break elif page_number > 100: break page_number += 1 with open(path, 'w', encoding='utf-8') as f: f.write(dumps(data))
from covid_crawlers.se_asia.jp_tokyo_data.tokyo_pdf_stats_map import stats_map, CITY _tokyo_cities_to_en = get_tokyo_cities_to_en_map() def pq(*args, **kw): for x in range(5): try: return _pq(*args, **kw) except: if x == 4: raise time.sleep(1) PDFS_BASE_DIR = get_overseas_dir() / 'jp_city_data' / 'pdfs' class ExtractFromTokyoPDF: def download_pdfs(self, only_most_recent=True): current_month = datetime.datetime.now().month if only_most_recent: months = [current_month] else: months = [ month for month in range(1, 13) ] for month in months: url = 'https://www.metro.tokyo.lg.jp/tosei/' \
def __init__(self): GithubRepo.__init__( self, output_dir=get_overseas_dir() / 'my' / 'covid-19-malaysia', github_url='https://github.com/ynshung/covid-19-malaysia') self.sdpf = StrictDataPointsFactory(region_mappings={ ('admin_0', '', 'malaysia'): ('admin_0', '', 'my'), ('admin_0', '', 'my'): ('admin_0', '', 'my'), ('admin_1', 'malaysia', 'johor'): ('admin_1', 'my', 'my-01'), ('admin_1', 'malaysia', 'kedah'): ('admin_1', 'my', 'my-02'), ('admin_1', 'malaysia', 'kelantan'): ('admin_1', 'my', 'my-03'), ('admin_1', 'malaysia', 'melaka'): ('admin_1', 'my', 'my-04'), ('admin_1', 'malaysia', 'negeri-sembilan'): None, ('admin_1', 'malaysia', 'pahang'): ('admin_1', 'my', 'my-06'), ('admin_1', 'malaysia', 'perak'): ('admin_1', 'my', 'my-08'), ('admin_1', 'malaysia', 'perlis'): ('admin_1', 'my', 'my-09'), ('admin_1', 'malaysia', 'pulau-pinang'): None, ('admin_1', 'malaysia', 'sabah'): ('admin_1', 'my', 'my-12'), ('admin_1', 'malaysia', 'sarawak'): ('admin_1', 'my', 'my-13'), ('admin_1', 'malaysia', 'selangor'): ('admin_1', 'my', 'my-10'), ('admin_1', 'malaysia', 'terengganu'): ('admin_1', 'my', 'my-11'), ('admin_1', 'malaysia', 'wp-kuala-lumpur'): None, ('admin_1', 'malaysia', 'wp-labuan'): None, ('admin_1', 'malaysia', 'wp-putrajaya'): None, ('admin_1', 'my', 'my-01'): ('admin_1', 'my', 'my-01'), ('admin_1', 'my', 'my-02'): ('admin_1', 'my', 'my-02'), ('admin_1', 'my', 'my-03'): ('admin_1', 'my', 'my-03'), ('admin_1', 'my', 'my-04'): ('admin_1', 'my', 'my-04'), ('admin_1', 'my', 'my-06'): ('admin_1', 'my', 'my-06'), ('admin_1', 'my', 'my-08'): ('admin_1', 'my', 'my-08'), ('admin_1', 'my', 'my-09'): ('admin_1', 'my', 'my-09'), ('admin_1', 'my', 'my-10'): ('admin_1', 'my', 'my-10'), ('admin_1', 'my', 'my-11'): ('admin_1', 'my', 'my-11'), ('admin_1', 'my', 'my-12'): ('admin_1', 'my', 'my-12'), ('admin_1', 'my', 'my-13'): ('admin_1', 'my', 'my-13'), ('my_district', 'my', 'alor-gajah'): None, ('my_district', 'my', 'asajaya'): ('my_district', 'my', 'asajaya'), ('my_district', 'my', 'bachok'): ('my_district', 'my', 'bachok'), ('my_district', 'my', 'baling'): ('my_district', 'my', 'baling'), ('my_district', 'my', 'bandar-bahru'): None, ('my_district', 'my', 'barat-daya'): None, ('my_district', 'my', 'batang-padang'): None, ('my_district', 'my', 'batu-pahat'): None, ('my_district', 'my', 'bau'): ('my_district', 'my', 'bau'), ('my_district', 'my', 'beaufort'): ('my_district', 'my', 'beaufort'), ('my_district', 'my', 'belaga'): ('my_district', 'my', 'belaga'), ('my_district', 'my', 'beluran'): ('my_district', 'my', 'beluran'), ('my_district', 'my', 'beluru'): None, ('my_district', 'my', 'bentong'): ('my_district', 'my', 'bentong'), ('my_district', 'my', 'bera'): ('my_district', 'my', 'bera'), ('my_district', 'my', 'besut'): ('my_district', 'my', 'besut'), ('my_district', 'my', 'betong'): ('my_district', 'my', 'betong'), ('my_district', 'my', 'bintulu'): ('my_district', 'my', 'bintulu'), ('my_district', 'my', 'bukit-mabong'): None, ('my_district', 'my', 'cameron'): None, ('my_district', 'my', 'cheras'): None, ('my_district', 'my', 'dalat'): ('my_district', 'my', 'dalat'), ('my_district', 'my', 'daro'): ('my_district', 'my', 'daro'), ('my_district', 'my', 'dungun'): ('my_district', 'my', 'dungun'), ('my_district', 'my', 'gombak'): ('my_district', 'my', 'gombak'), ('my_district', 'my', 'gua-musang'): None, ('my_district', 'my', 'hillir-perak'): None, ('my_district', 'my', 'hulu-langat'): None, ('my_district', 'my', 'hulu-perak'): None, ('my_district', 'my', 'hulu-selangor'): None, ('my_district', 'my', 'hulu-terengganu'): None, ('my_district', 'my', 'jasin'): ('my_district', 'my', 'jasin'), ('my_district', 'my', 'jelebu'): ('my_district', 'my', 'jelebu'), ('my_district', 'my', 'jeli'): ('my_district', 'my', 'jeli'), ('my_district', 'my', 'jempol'): ('my_district', 'my', 'jempol'), ('my_district', 'my', 'jerantut'): ('my_district', 'my', 'jerantut'), ('my_district', 'my', 'johor-bahru'): None, ('my_district', 'my', 'julau'): ('my_district', 'my', 'julau'), ('my_district', 'my', 'kabong'): None, ('my_district', 'my', 'kampar'): ('my_district', 'my', 'kampar'), ('my_district', 'my', 'kanowit'): ('my_district', 'my', 'kanowit'), ('my_district', 'my', 'kapit'): ('my_district', 'my', 'kapit'), ('my_district', 'my', 'kemaman'): ('my_district', 'my', 'kemaman'), ('my_district', 'my', 'keningau'): ('my_district', 'my', 'keningau'), ('my_district', 'my', 'kepong'): None, ('my_district', 'my', 'kerian'): ('my_district', 'my', 'kerian'), ('my_district', 'my', 'kinabatangan'): ('my_district', 'my', 'kinabatangan'), ('my_district', 'my', 'kinta'): ('my_district', 'my', 'kinta'), ('my_district', 'my', 'klang'): ('my_district', 'my', 'klang'), ('my_district', 'my', 'kluang'): ('my_district', 'my', 'kluang'), ('my_district', 'my', 'kota-belud'): None, ('my_district', 'my', 'kota-bharu'): None, ('my_district', 'my', 'kota-kinabalu'): None, ('my_district', 'my', 'kota-marudu'): None, ('my_district', 'my', 'kota-samarahan'): None, ('my_district', 'my', 'kota-setar'): None, ('my_district', 'my', 'kota-tinggi'): None, ('my_district', 'my', 'kuala-kangsar'): None, ('my_district', 'my', 'kuala-krai'): None, ('my_district', 'my', 'kuala-langat'): None, ('my_district', 'my', 'kuala-muda'): None, ('my_district', 'my', 'kuala-nerus'): None, ('my_district', 'my', 'kuala-penyu'): None, ('my_district', 'my', 'kuala-pilah'): None, ('my_district', 'my', 'kuala-selangor'): None, ('my_district', 'my', 'kuala-terengganu'): None, ('my_district', 'my', 'kuantan'): ('my_district', 'my', 'kuantan'), ('my_district', 'my', 'kubang-pasu'): None, ('my_district', 'my', 'kuching'): ('my_district', 'my', 'kuching'), ('my_district', 'my', 'kudat'): ('my_district', 'my', 'kudat'), ('my_district', 'my', 'kulai'): None, ('my_district', 'my', 'kulim'): ('my_district', 'my', 'kulim'), ('my_district', 'my', 'kunak'): ('my_district', 'my', 'kunak'), ('my_district', 'my', 'lahad-datu'): None, ('my_district', 'my', 'langkawi'): ('my_district', 'my', 'langkawi'), ('my_district', 'my', 'larut-matang-selama'): None, ('my_district', 'my', 'lawas'): ('my_district', 'my', 'lawas'), ('my_district', 'my', 'lembah-pantai'): None, ('my_district', 'my', 'limbang'): ('my_district', 'my', 'limbang'), ('my_district', 'my', 'lipis'): ('my_district', 'my', 'lipis'), ('my_district', 'my', 'lubok-antu'): None, ('my_district', 'my', 'lundu'): ('my_district', 'my', 'lundu'), ('my_district', 'my', 'machang'): ('my_district', 'my', 'machang'), ('my_district', 'my', 'manjung'): None, ('my_district', 'my', 'maran'): ('my_district', 'my', 'maran'), ('my_district', 'my', 'marang'): ('my_district', 'my', 'marang'), ('my_district', 'my', 'marudi'): ('my_district', 'my', 'marudi'), ('my_district', 'my', 'matu'): ('my_district', 'my', 'matu'), ('my_district', 'my', 'melaka-tengah'): None, ('my_district', 'my', 'meradong'): ('my_district', 'my', 'meradong'), ('my_district', 'my', 'mersing'): ('my_district', 'my', 'mersing'), ('my_district', 'my', 'miri'): ('my_district', 'my', 'miri'), ('my_district', 'my', 'muallim'): None, ('my_district', 'my', 'muar'): ('my_district', 'my', 'muar'), ('my_district', 'my', 'mukah'): ('my_district', 'my', 'mukah'), ('my_district', 'my', 'nabawan'): ('my_district', 'my', 'nabawan'), ('my_district', 'my', 'padang-terap'): None, ('my_district', 'my', 'pakan'): ('my_district', 'my', 'pakan'), ('my_district', 'my', 'papar'): ('my_district', 'my', 'papar'), ('my_district', 'my', 'pasir-mas'): None, ('my_district', 'my', 'pasir-putih'): None, ('my_district', 'my', 'pekan'): ('my_district', 'my', 'pekan'), ('my_district', 'my', 'penampang'): ('my_district', 'my', 'penampang'), ('my_district', 'my', 'pendang'): ('my_district', 'my', 'pendang'), ('my_district', 'my', 'perak-tengah'): None, ('my_district', 'my', 'petaling'): ('my_district', 'my', 'petaling'), ('my_district', 'my', 'pitas'): ('my_district', 'my', 'pitas'), ('my_district', 'my', 'pontian'): ('my_district', 'my', 'pontian'), ('my_district', 'my', 'port-dickson'): None, ('my_district', 'my', 'pusa'): None, ('my_district', 'my', 'putatan'): ('my_district', 'my', 'putatan'), ('my_district', 'my', 'ranau'): ('my_district', 'my', 'ranau'), ('my_district', 'my', 'raub'): ('my_district', 'my', 'raub'), ('my_district', 'my', 'rembau'): ('my_district', 'my', 'rembau'), ('my_district', 'my', 'rompin'): ('my_district', 'my', 'rompin'), ('my_district', 'my', 'sabak-bernam'): None, ('my_district', 'my', 'sandakan'): ('my_district', 'my', 'sandakan'), ('my_district', 'my', 'saratok'): ('my_district', 'my', 'saratok'), ('my_district', 'my', 'sarikei'): ('my_district', 'my', 'sarikei'), ('my_district', 'my', 'sebauh'): None, ('my_district', 'my', 'seberang-perai-selatan'): None, ('my_district', 'my', 'seberang-perai-tengah'): None, ('my_district', 'my', 'seberang-perai-utara'): None, ('my_district', 'my', 'segamat'): ('my_district', 'my', 'segamat'), ('my_district', 'my', 'selangau'): ('my_district', 'my', 'selangau'), ('my_district', 'my', 'semporna'): ('my_district', 'my', 'semporna'), ('my_district', 'my', 'sepang'): ('my_district', 'my', 'sepang'), ('my_district', 'my', 'seremban'): ('my_district', 'my', 'seremban'), ('my_district', 'my', 'serian'): ('my_district', 'my', 'serian'), ('my_district', 'my', 'setiu'): ('my_district', 'my', 'setiu'), ('my_district', 'my', 'sibu'): ('my_district', 'my', 'sibu'), ('my_district', 'my', 'sik'): ('my_district', 'my', 'sik'), ('my_district', 'my', 'simunjan'): ('my_district', 'my', 'simunjan'), ('my_district', 'my', 'sipitang'): ('my_district', 'my', 'sipitang'), ('my_district', 'my', 'song'): ('my_district', 'my', 'song'), ('my_district', 'my', 'sri-aman'): None, ('my_district', 'my', 'subis'): None, ('my_district', 'my', 'tambunan'): ('my_district', 'my', 'tambunan'), ('my_district', 'my', 'tampin'): ('my_district', 'my', 'tampin'), ('my_district', 'my', 'tanah-merah'): None, ('my_district', 'my', 'tangkak'): None, ('my_district', 'my', 'tanjung-manis'): None, ('my_district', 'my', 'tatau'): ('my_district', 'my', 'tatau'), ('my_district', 'my', 'tawau'): ('my_district', 'my', 'tawau'), ('my_district', 'my', 'tebedu'): None, ('my_district', 'my', 'telung-usan'): None, ('my_district', 'my', 'temerloh'): ('my_district', 'my', 'temerloh'), ('my_district', 'my', 'tenom'): ('my_district', 'my', 'tenom'), ('my_district', 'my', 'timur-laut'): None, ('my_district', 'my', 'titiwangsa'): None, ('my_district', 'my', 'tongod'): ('my_district', 'my', 'tongod'), ('my_district', 'my', 'tuaran'): ('my_district', 'my', 'tuaran'), ('my_district', 'my', 'tumpat'): ('my_district', 'my', 'tumpat'), ('my_district', 'my', 'yan'): ('my_district', 'my', 'yan') }, mode=MODE_STRICT) self.update()
def _get_datapoints(self): r = self.sdpf() for date in self.iter_nonempty_dirs(self.output_dir): for province_dict in provinces: if province_dict["value"] in ('86', '88', '710000', '810000'): continue path = get_overseas_dir( ) / 'cn' / 'data' / date / f'{province_dict["value"]}.json' print(path) with open(path, 'r', encoding='utf-8') as f: json_data = f.read() json_data = json_data.replace( "dojo_request_script_callbacks.dojo_request_script0(", "") json_data = json_data.rstrip().rstrip(");") data = json.loads(json_data) # {"author":"geodata","city":"河南省","code":"410000","codeis":"410000", # "country":"中国","createAt":1599211044000,"day":"2020-09-03", # "id":"43030b8e44d35d00af7a25deb55524a4", # "nconfirm":0,"ndead":0,"nheal":0,"nsuspect":0,"params":{}, # "province":"河南","publishAt":1599148740000, # "tconfirm":1276,"tdead":22,"theal":1254,"tsuspect":0, # "updateAt":1599211044000} for item in data: i_date = self.convert_date(item['day']) r.append( region_schema=Schemas.CN_CITY, region_parent='CN', #item['province'], region_child=item['city'], datatype=DataTypes.TOTAL, value=int(item[f'tconfirm']) + int(item[f'tsuspect']), date_updated=i_date, source_url=self.SOURCE_URL) r.append( region_schema=Schemas.CN_CITY, region_parent='CN', #item['province'], region_child=item['city'], datatype=DataTypes.NEW, value=int(item[f'nconfirm']) + int(item[f'nsuspect']), date_updated=i_date, source_url=self.SOURCE_URL) for total_datatype, new_datatype, key_suffix in ( (DataTypes.CONFIRMED, DataTypes.CONFIRMED_NEW, 'confirm'), (DataTypes.PROBABLE, DataTypes.PROBABLE_NEW, 'suspect'), (DataTypes.STATUS_RECOVERED, DataTypes.STATUS_RECOVERED_NEW, 'heal'), (DataTypes.STATUS_DEATHS, DataTypes.STATUS_DEATHS_NEW, 'dead'), ): r.append( region_schema=Schemas.CN_CITY, region_parent='CN', #item['province'], region_child=item['city'], datatype=total_datatype, value=int(item[f'n{key_suffix}']), date_updated=i_date, source_url=self.SOURCE_URL) r.append( region_schema=Schemas.CN_CITY, region_parent='CN', #item['province'], region_child=item['city'], datatype=new_datatype, value=int(item[f't{key_suffix}']), date_updated=i_date, source_url=self.SOURCE_URL) return r
def _get_datapoints(self): # dateRep day month year cases deaths countriesAndTerritories geoId countryterritoryCode # popData2019 continentExp Cumulative_number_for_14_days_of_COVID-19_cases_per_100000 #s # 25/08/2020 25 8 2020 71 10 Afghanistan AF AFG 38041757 Asia 2.67074941 # 24/08/2020 24 8 2020 0 0 Afghanistan AF AFG 38041757 Asia 2.48411239 # 23/08/2020 23 8 2020 105 2 Afghanistan AF AFG 38041757 Asia 2.48411239 # 22/08/2020 22 8 2020 38 0 Afghanistan AF AFG 38041757 Asia 2.31061883 # 21/08/2020 21 8 2020 97 2 Afghanistan AF AFG 38041757 Asia 2.41576644 # 20/08/2020 20 8 2020 160 8 Afghanistan AF AFG 38041757 Asia 2.26855978 r = self.sdpf() date = datetime.today().strftime('%Y_%m_%d') df = pd.read_excel(get_overseas_dir() / 'world_eu_cdc' / 'data' / date / 'world_data.xlsx', engine="openpyxl", parse_dates=False) date_rep_out = [] for idx, item in df.iterrows(): if '/' in item['dateRep']: dd, mm, yyyy = item['dateRep'].split('/') date_rep_out.append(f'{yyyy}/{mm}/{dd}') else: date_rep_out.append(item['dateRep']) df['dateRep'] = date_rep_out df = df.sort_values('dateRep', axis=0) prev_date = None cases_cumulative = Counter() deaths_cumulative = Counter() for idx, item in df.iterrows(): date = self.convert_date(str(item['dateRep']).split()[0]) if item['geoId'] in ('BQ', 'JPG11668'): continue elif item['geoId'] == 'EL': item['geoId'] = 'gr' elif item['geoId'] == 'UK': item['geoId'] = 'gb' elif not isinstance(item['geoId'], str): continue if prev_date and prev_date > date: raise Exception(prev_date, date) prev_date = date cases_cumulative[item['geoId']] += int(item['cases']) deaths_cumulative[item['geoId']] += int(item['deaths']) r.append(region_schema=Schemas.ADMIN_0, region_parent='', region_child=item['geoId'], datatype=DataTypes.TOTAL, value=cases_cumulative[item['geoId']], date_updated=date, source_url=self.SOURCE_URL) r.append(region_schema=Schemas.ADMIN_0, region_parent='', region_child=item['geoId'], datatype=DataTypes.STATUS_DEATHS, value=deaths_cumulative[item['geoId']], date_updated=date, source_url=self.SOURCE_URL) return r
def __init__(self): GithubRepo.__init__(self, output_dir=get_overseas_dir() / 'fr' / 'data', github_url='https://github.com/opencovid19-fr/data') self.sdpf = StrictDataPointsFactory(mode=MODE_STRICT) self.update()
def __init__(self): # Only raw_data4.json is currently being updated, # so won't download the others every day URLBase.__init__( self, output_dir=get_overseas_dir() / 'sn' / 'data', urls_dict={ 'by_district.csv': URL( 'https://docs.google.com/spreadsheets/d/e/' '2PACX-1vRj1sRWYmyZ2AznFdP5Dr98uZrzsMMudPBRIcMW8FdwAEy-' 'Hwq3PSPJYI12xTzLbA/pub?gid=1515611831&single=true&output=csv', static_file=False), 'national_cases.csv': URL( 'https://docs.google.com/spreadsheets/d/e/' '2PACX-1vRj1sRWYmyZ2AznFdP5Dr98uZrzsMMudPBRIcMW8FdwAEy-' 'Hwq3PSPJYI12xTzLbA/pub?gid=708820609&single=true&output=csv', static_file=False) }) self.sdpf = StrictDataPointsFactory( region_mappings={ ('admin_1', 'sn', 'dakar-ouest'): ('MERGE', 'admin_1', 'sn', 'sn-dk'), ('admin_1', 'sn', 'dakar'): ('MERGE', 'admin_1', 'sn', 'sn-dk'), ('admin_1', 'sn', 'dakar-centre'): ('MERGE', 'admin_1', 'sn', 'sn-dk'), ('admin_1', 'sn', 'dakar-nord'): ('MERGE', 'admin_1', 'sn', 'sn-dk'), ('admin_1', 'sn', 'dakar-sud'): ('MERGE', 'admin_1', 'sn', 'sn-dk'), ('admin_1', 'sn', 'mbao'): ('MERGE', 'admin_1', 'sn', 'sn-dk'), ('admin_1', 'sn', 'guédiawaye'): ('MERGE', 'admin_1', 'sn', 'sn-dk'), ('admin_1', 'sn', 'yeumbeul'): ('MERGE', 'admin_1', 'sn', 'sn-dk'), ('admin_1', 'sn', 'rufisque'): ('MERGE', 'admin_1', 'sn', 'sn-dk'), ('admin_1', 'sn', 'diamniadio'): ('MERGE', 'admin_1', 'sn', 'sn-dk'), ('admin_1', 'sn', 'pikine'): ('MERGE', 'admin_1', 'sn', 'sn-dk'), ('admin_1', 'sn', 'sangalkam'): ('MERGE', 'admin_1', 'sn', 'sn-dk'), ('admin_1', 'sn', 'keur massar'): ('MERGE', 'admin_1', 'sn', 'sn-dk'), ('admin_1', 'sn', 'guediawaye'): ('MERGE', 'admin_1', 'sn', 'sn-dk'), ('admin_1', 'sn', 'touba'): ('MERGE', 'admin_1', 'sn', 'sn-db'), ('admin_1', 'sn', 'mbour'): ('MERGE', 'admin_1', 'sn', 'sn-th'), ('admin_1', 'sn', 'popenguine'): ('MERGE', 'admin_1', 'sn', 'sn-th'), ('admin_1', 'sn', 'oussouye'): ('MERGE', 'admin_1', 'sn', 'sn-zg'), ('admin_1', 'sn', 'saint louis'): ('MERGE', 'admin_1', 'sn', 'sn-sl'), ('admin_1', 'sn', 'richard toll'): ('MERGE', 'admin_1', 'sn', 'sn-sl'), ('admin_1', 'sn', 'non déterminé'): ('MERGE', 'admin_1', 'sn', 'unknown'), ('admin_1', 'sn', ''): ('MERGE', 'admin_1', 'sn', 'unknown'), ('admin_1', 'sn', 'velingara'): ('MERGE', 'admin_1', 'sn', 'sn-kd'), ('admin_1', 'sn', 'goudiry'): ('MERGE', 'admin_1', 'sn', 'sn-tc'), ('admin_1', 'sn', 'keur massar'): ('MERGE', 'admin_1', 'sn', 'sn-tc'), ('admin_1', 'sn', 'sakal'): ('MERGE', 'admin_1', 'sn', 'sn-lg'), ('admin_1', 'sn', 'mbacke'): ('MERGE', 'admin_1', 'sn', 'sn-db'), ('admin_1', 'sn', 'tivaouane'): ('MERGE', 'admin_1', 'sn', 'sn-th'), ('admin_1', 'sn', 'pout'): ('MERGE', 'admin_1', 'sn', 'sn-th'), ('admin_1', 'sn', 'nioro du rip'): ('MERGE', 'admin_1', 'sn', 'sn-kl'), ('admin_1', 'sn', 'sedhiou'): ('MERGE', 'admin_1', 'sn', 'sn-se'), ('admin_1', 'sn', 'diouloulou'): ('MERGE', 'admin_1', 'sn', 'sn-zg'), ('admin_1', 'sn', 'linguere'): ('MERGE', 'admin_1', 'sn', 'sn-lg'), ('admin_1', 'sn', 'kedougou'): ('MERGE', 'admin_1', 'sn', 'sn-ke'), ('admin_1', 'sn', 'thiadiaye'): ('MERGE', 'admin_1', 'sn', 'sn-th'), ('admin_1', 'sn', 'mekhe'): ('MERGE', 'admin_1', 'sn', 'sn-th'), ('admin_1', 'sn', 'koungheul'): ('MERGE', 'admin_1', 'sn', 'sn-ka'), ('admin_1', 'sn', 'coki'): ('MERGE', 'admin_1', 'sn', 'sn-lg'), ('admin_1', 'sn', 'poponguine'): ('MERGE', 'admin_1', 'sn', 'sn-th'), ('admin_1', 'sn', 'tivaoune'): ('MERGE', 'admin_1', 'sn', 'sn-th'), ('admin_1', 'sn', 'joal fadiouth'): ('MERGE', 'admin_1', 'sn', 'sn-th'), ('admin_1', 'sn', 'joal-fadiouth'): ('MERGE', 'admin_1', 'sn', 'sn-th'), ('admin_1', 'sn', 'bignona'): ('MERGE', 'admin_1', 'sn', 'sn-zg'), ('admin_1', 'sn', 'bikilane'): ('MERGE', 'admin_1', 'sn', 'sn-dk'), # Pikine?? ('admin_1', 'sn', 'dioffior'): ('MERGE', 'admin_1', 'sn', 'sn-fk'), ('admin_1', 'sn', 'darou mousty'): ('MERGE', 'admin_1', 'sn', 'sn-lg'), ('admin_1', 'sn', 'thilogne'): ('MERGE', 'admin_1', 'sn', 'sn-mt'), ('admin_1', 'sn', 'bounkiling'): ('MERGE', 'admin_1', 'sn', 'sn-se'), ('admin_1', 'sn', 'khombole'): ('MERGE', 'admin_1', 'sn', 'sn-th'), ('admin_1', 'sn', 'richard-toll'): ('MERGE', 'admin_1', 'sn', 'sn-sl'), ('admin_1', 'sn', 'birkilane'): ('MERGE', 'admin_1', 'sn', 'sn-ka'), ('admin_1', 'sn', 'kebemer'): ('MERGE', 'admin_1', 'sn', 'sn-lg'), ('admin_1', 'sn', 'ranerou'): ('MERGE', 'admin_1', 'sn', 'sn-mt'), ('admin_1', 'sn', 'gossas'): ('MERGE', 'admin_1', 'sn', 'sn-fk'), ('admin_1', 'sn', 'bambey'): ('MERGE', 'admin_1', 'sn', 'sn-db'), ('admin_1', 'sn', 'sokone'): ('MERGE', 'admin_1', 'sn', 'sn-fk'), ('admin_1', 'sn', 'mékhé'): ('MERGE', 'admin_1', 'sn', 'sn-th'), ('admin_1', 'sn', 'joal'): ('MERGE', 'admin_1', 'sn', 'sn-th'), ('admin_1', 'sn', 'mbacké'): ('MERGE', 'admin_1', 'sn', 'sn-db'), ('admin_1', 'sn', 'vélingara'): ('MERGE', 'admin_1', 'sn', 'sn-kd'), ('admin_1', 'sn', 'bounkilig'): ('MERGE', 'admin_1', 'sn', 'sn-se'), ('admin_1', 'sn', 'diakhao'): ('MERGE', 'admin_1', 'sn', 'sn-fk'), ('admin_1', 'sn', 'linguère'): ('MERGE', 'admin_1', 'sn', 'sn-lg'), ('admin_1', 'sn', 'cocki'): ('MERGE', 'admin_1', 'sn', 'sn-lg'), ('admin_1', 'sn', 'kébémer'): ('MERGE', 'admin_1', 'sn', 'sn-lg'), ('admin_1', 'sn', 'nioro'): ('MERGE', 'admin_1', 'sn', 'sn-kl'), ('admin_1', 'sn', 'ranérou'): ('MERGE', 'admin_1', 'sn', 'sn-mt'), ('admin_1', 'sn', 'kanel'): ('MERGE', 'admin_1', 'sn', 'sn-mt'), ('admin_1', 'sn', 'joal fadiouth'): ('MERGE', 'admin_1', 'sn', 'sn-th'), ('admin_1', 'sn', 'passy'): ('MERGE', 'admin_1', 'sn', 'sn-fk'), }, mode=MODE_STRICT) self.update()
def get_datapoints(self): r = [] for date in listdir(get_overseas_dir() / 'world_um' / 'api_data'): print(date) r.extend(self._get_datapoints(date)) return r
def _get_datapoints(self, date): r = self.sdpf() path = get_overseas_dir( ) / 'world_um' / 'api_data' / date / 'data.json' print(path) with open(path, 'r', encoding='utf-8') as f: data = loads(f.read()) #print(data) for country, country_data_dict in data['covid'].items(): for country_dict in country_data_dict['countries']: key = 'smoothed_cli' if 'smoothed_cli' in country_dict else 'smoothed_covid_se' if country_dict[key] is None: continue print(country_dict) r.append(region_schema=Schemas.ADMIN_0, region_parent='', region_child=country, datatype=DataTypes.FACEBOOK_COVID_SYMPTOMS, value=round(country_dict[key] * 100000), date_updated=self.convert_date( country_dict['survey_date'], formats=('%Y%m%d', )), source_url=self.SOURCE_URL) for region_dict in country_data_dict['regions']: key = 'smoothed_cli' if 'smoothed_cli' in country_dict else 'smoothed_covid_se' if region_dict[key] is None: continue print(region_dict) r.append(region_schema=Schemas.ADMIN_1, region_parent=country, region_child=region_dict['region'], datatype=DataTypes.FACEBOOK_COVID_SYMPTOMS, value=round(region_dict[key] * 100000), date_updated=self.convert_date( region_dict['survey_date'], formats=('%Y%m%d', )), source_url=self.SOURCE_URL) for country, country_data_dict in data['flu'].items(): for country_dict in country_data_dict['countries']: key = 'smoothed_ili' if 'smoothed_ili' in country_dict else 'smoothed_flu_se' if country_dict[key] is None: continue print(country_dict) r.append(region_schema=Schemas.ADMIN_0, region_parent='', region_child=country, datatype=DataTypes.FACEBOOK_FLU_SYMPTOMS, value=round(country_dict[key] * 100000), date_updated=self.convert_date( country_dict['survey_date'], formats=('%Y%m%d', )), source_url=self.SOURCE_URL) for region_dict in country_data_dict['regions']: key = 'smoothed_ili' if 'smoothed_ili' in country_dict else 'smoothed_flu_se' if region_dict[key] is None: continue print(region_dict) r.append(region_schema=Schemas.ADMIN_1, region_parent=country, region_child=region_dict['region'], datatype=DataTypes.FACEBOOK_FLU_SYMPTOMS, value=round(region_dict[key] * 100000), date_updated=self.convert_date( region_dict['survey_date'], formats=('%Y%m%d', )), source_url=self.SOURCE_URL) return r
def __init__(self): # Only raw_data4.json is currently being updated, # so won't download the others every day GlobalBase.__init__(self, output_dir=get_overseas_dir() / 'rs' / 'data') self.__today_dir = self.output_dir / datetime.datetime.now().strftime('%Y_%m_%d') self.update()
def _get_datapoints(self): # country_region_code,country_region,sub_region_1,sub_region_2,metro_area,iso_3166_2_code,census_fips_code, # date, # retail_and_recreation_percent_change_from_baseline, # grocery_and_pharmacy_percent_change_from_baseline, # parks_percent_change_from_baseline, # transit_stations_percent_change_from_baseline, # workplaces_percent_change_from_baseline, # residential_percent_change_from_baseline # AE,United Arab Emirates,,,,,,2020-02-15,0,4,5,0,2,1 # AE,United Arab Emirates,,,,,,2020-02-16,1,4,4,1,2,1 # AE,United Arab Emirates,,,,,,2020-02-17,-1,1,5,1,2,1 # AE,United Arab Emirates,,,,,,2020-02-18,-2,1,5,0,2,1 r = self.sdpf() date = datetime.today().strftime('%Y_%m_%d') path = get_overseas_dir( ) / 'world_google_mobility' / 'data' / date / 'mobility.csv' with open(path, 'r', encoding='utf-8') as f: for item in csv.DictReader(f): date = self.convert_date(item['date']) if item['iso_3166_2_code']: region_schema = Schemas.ADMIN_1 region_parent = item['country_region_code'] region_child = item['iso_3166_2_code'] elif item['sub_region_2'] and item[ 'country_region_code'] == 'AU': region_schema = Schemas.LGA region_parent = item['sub_region_1'] region_child = normalize_locality_name( item['sub_region_2']) elif not item['sub_region_1'] and not item['sub_region_2']: region_schema = Schemas.ADMIN_0 region_parent = '' region_child = item['country_region_code'] else: continue for datatype, value in ( (DataTypes.GOOGLE_MOBILITY_RETAIL_RECREATION, item['retail_and_recreation_percent_change_from_baseline'] ), (DataTypes.GOOGLE_MOBILITY_SUPERMARKET_PHARMACY, item['grocery_and_pharmacy_percent_change_from_baseline'] ), (DataTypes.GOOGLE_MOBILITY_PARKS, item['parks_percent_change_from_baseline']), (DataTypes.GOOGLE_MOBILITY_PUBLIC_TRANSPORT, item['transit_stations_percent_change_from_baseline']), (DataTypes.GOOGLE_MOBILITY_WORKPLACES, item['workplaces_percent_change_from_baseline']), (DataTypes.GOOGLE_MOBILITY_RESIDENTIAL, item['residential_percent_change_from_baseline'])): if not value: continue r.append(region_schema=region_schema, region_parent=region_parent, region_child=region_child, datatype=datatype, value=int(value), date_updated=date, source_url=self.SOURCE_URL) return r