def load_data():
""" Get hospital data from OHA """
# with open("./oha.html", "r", encoding="utf-8") as fp:
# return fp.read()
gateway = HTMLGateway()
return gateway.fetch(url)
def download_zip(url, targetdir, updated):
result = False
(urlpart, fname) = os.path.split(url)
local_filename = os.path.join(targetdir, fname)
overwrite = False
agencytxt = os.path.join(targetdir, 'agency.txt')
current = None
if os.path.exists(agencytxt):
current_time = os.stat(agencytxt).st_ctime
current = datetime.fromtimestamp(current_time)
if not updated or not current or (current < updated):
print("Time to get a new copy for \"%s\"" % targetdir)
overwrite = True
if overwrite or not os.path.exists(local_filename):
if verbose:
print("Downloading \"%s\" to \"%s\"." % (url, targetdir))
try:
with HTMLGateway.stream(url) as data:
if not os.path.exists(targetdir): os.mkdir(targetdir)
with open(local_filename, 'wb') as fp:
shutil.copyfileobj(data.raw, fp)
result = True
except Exception as e:
print("Download failed. %s" % e)
# elif verbose:
# print("Already have \"%s\", skipping download." % local_filename)
return result
def get_table(site):
""" Scrapes the datatable from the front page of the website """
raw_data = HTMLGateway.fetch(site)
# Find the first table in the page.
soup = BeautifulSoup(raw_data, features="lxml")
rows = soup.find_all('tr')
keys = []
table = []
for row in rows:
headers = row.findAll('th')
if headers:
for item in headers:
try:
keys.append(item.get_text().strip())
except Exception as e:
pass
else:
values = {}
columns = row.findAll('td')
if len(columns) != len(keys):
continue
for i in range(0, len(keys)):
values[keys[i]] = columns[i].get_text().strip()
table.append(values)
return table
def last_feature_edit():
"""
Fetch the JSON for the OHA feature layer.
@Returns:
Last edit time (datetime object in UTC)
"""
ohajson = HTMLGateway.fetch(featureLayerUrl + '?f=json')
oha = json.loads(ohajson)
# Convert from the Esri format to the real one by /1000
unixtime = int(oha['editingInfo']['lastEditDate']) / 1000
utc = datetime.utcfromtimestamp(unixtime)
return utc
try:
attributes["total_recovered"] = s2i(df.at['Total Recovered'])
except KeyError:
pass
n = {"attributes": attributes, "geometry": geometry}
#print(n)
results = layer.edit_features(adds=[n])
return results['addResults'][0]['success']
#============================================================================
if __name__ == "__main__":
world_data = None
try:
world_data = HTMLGateway.fetch(worldometer_world_url)
except Exception as e:
print("Could not fetch world data.", e)
exit(-1)
parser = WorldometerParser()
# Convert the data into a DataFrame
world_df = parser.create_df(world_data, "main_table_countries_today", '1')
world_last_updated = parser.parse_last_updated(world_data)
state_data = None
try:
state_data = HTMLGateway.fetch(worldometer_states_url)
except Exception as e:
print("Could not fetch state data.", e)
raise Exception("Typo fix please %s" % e)
#print(n)
print(df)
results = layer.edit_features(adds=[n])
return results['addResults'][0]['success']
#============================================================================
if __name__ == "__main__":
url = Config.WA_URL
# Get data from the wide world of web
try:
gateway = HTMLGateway()
raw_data = gateway.fetch(url)
except Exception as e:
print("Could not fetch data.", e)
exit(-1)
# Convert the data into a DataFrame
parser = WAParser()
last_updated = parser.parse_last_updated(raw_data)
df = parser.fetch_cases(raw_data)
# Open portal to make sure it's there!
try:
portal = GIS(portalUrl, portalUser, portalPasswd)
#print("Logged in as " + str(portal.properties.user.username))
layer = connect(portal, featurelayerUrl)