def azure(self, addr, local, country, saveraw):
output = self.initOutput()
# create query
address = "" if addr is None else addr
address = address + ("" if local is None else "," + local)
address = address + ("" if country is None else "," + country)
# init service if not init yet
if not self.geolocator_azure:
self.geolocator_azure = AzureMaps(
subscription_key=self.SERVICES[self.CURRENT_SERVICE]['key'])
# geocode address
location = self.geolocator_azure.geocode(address,
exactly_one=False,
language="pt-PT")
if location is not None:
answer = location[0].raw
output['status'] = "OK"
output["latitude"] = location[0].latitude
output["longitude"] = location[0].longitude
output["number_of_results"] = len(location)
output["input_string"] = address
output["accuracy"] = answer.get('score')
output["place_id"] = answer.get("id")
if answer.get("address"):
output["formatted_address"] = answer.get('address').get(
'freeformAddress')
output["distrito"] = answer.get("address").get(
"countrySubdivision")
# maybe?
output["concelho"] = answer.get("address").get("municipality")
output["freguesia"] = answer.get("address").get(
"municipalitySubdivision")
CPext = answer.get("address").get('extendedPostalCode')
CP = answer.get("address").get('postalCode')
if CPext:
CPext = CPext.split(',')[0]
CPext = CPext[:4] + '-' + CPext[4:]
output["postcode"] = CPext
elif CP:
output["postcode"] = CP.split(',')[0]
output["type"] = answer.get('type')
output["service"] = self.SERVICES[self.CURRENT_SERVICE]['service']
if saveraw:
output["response"] = location[0].raw
else:
output['status'] = "ZERO_RESULTS"
return output
def __init__(self, config: DefaultConfig):
luis_application = LuisApplication(
config.LUIS_APP_ID,
config.LUIS_API_KEY,
"https://" + config.LUIS_API_HOST_NAME,
)
luis_options = LuisPredictionOptions(
include_all_intents=True, include_instance_data=True
)
self.recognizer = LuisRecognizer(luis_application, luis_options, True)
self.fetch_dataset()
self._AzMap = AzureMaps(subscription_key=config.AZURE_MAPS_KEY)
class HeroBot(ActivityHandler):
def __init__(self, config: DefaultConfig):
luis_application = LuisApplication(
config.LUIS_APP_ID,
config.LUIS_API_KEY,
"https://" + config.LUIS_API_HOST_NAME,
)
luis_options = LuisPredictionOptions(
include_all_intents=True, include_instance_data=True
)
self.recognizer = LuisRecognizer(luis_application, luis_options, True)
self._last_file_name = None
self._last_date = None
self._last_file_update = None
self.fetch_dataset()
self._AzMap = AzureMaps(subscription_key=config.AZURE_MAPS_KEY)
def fetch_dataset(self, force = False):
last_file_name, last_date = self._get_last_file_name()
last_file_update = self._get_last_update(last_file_name)
if (self._last_file_name is None) or (last_date > self._last_date) \
or (last_file_update > self._last_file_update) or force:
self._data = pd.read_csv(C.FILE_URL_BASE + last_file_name)
self._last_file_name = last_file_name
self._last_date = last_date
self._last_file_update = last_file_update
log.info(f"Updated dataset, new last_date = {self._last_date}, last committed = {self._last_file_update}")
else:
log.debug(f"Based on timestamp check, last_date = {last_date}, prev last_update = {self._last_date}, " +
f"timestamps: old {self._last_file_update}, new: {last_file_update} , no refresh required")
def _get_last_file_name(self):
ret = (None, None)
try:
req = requests.get(C.FILE_DIR_URL)
dts = pd.DataFrame(pd.Series([n["name"] for n in req.json()], name="name"))
dts["dt"] = pd.to_datetime(dts["name"].str.rstrip(".csv"), errors = "coerce")
last_file = dts.sort_values("dt", ascending=False)["name"].tolist()[0]
last_date = dts.sort_values("dt", ascending=False)["dt"].tolist()[0]
ret = (last_file, last_date)
except Exception as e:
log.error(f"Error getting last filename and date, message: {e}")
return ret
def _get_last_update(self, filename):
ret = None
if filename is not None:
try:
req = requests.get(C.LAST_UPDATE_URL_TEMPLATE.format(filename))
last_update = req.json()[0]["commit"]["committer"]["date"]
ret = pd.to_datetime(last_update)
except Exception as e:
log.error(f"Getting the last update timestamp failed with message: {e}, timestamp is set to: {self._last_file_update}")
return ret
def _filter_by_cntry(self, cntry):
df = (self._data
.query("Country_Region == @cntry")
.groupby("Country_Region")[['Confirmed', 'Deaths', 'Recovered', 'Active']]
.sum())
if df.shape[0] == 0:
log.warning(f"Encountered country matching problem, Country = {cntry}")
else:
confirmed, dead, recovered, active = df.values[0]
return (confirmed, dead, recovered, active )
async def on_members_added_activity(
self, members_added: [ChannelAccount], turn_context: TurnContext
):
for member in members_added:
if member.id != turn_context.activity.recipient.id:
card = HeroCard(
title="Welcome to the COVID-19 Information bot",
images=[
CardImage(
url="https://i.imgur.com/zm095AG.png"
)
],
buttons=[
CardAction(
type=ActionTypes.open_url,
title="Repository link",
value="https://github.com/vykhand/realherobot",
)
],
)
repl = MessageFactory.list([])
repl.attachments.append(CardFactory.hero_card(card))
await turn_context.send_activity(repl)
async def on_message_activity(self, turn_context: TurnContext):
# First, we use the dispatch model to determine which cognitive service (LUIS or QnA) to use.
recognizer_result = await self.recognizer.recognize(turn_context)
# Top intent tell us which cognitive service to use.
intent = LuisRecognizer.top_intent(recognizer_result)
# Next, we call the dispatcher with the top intent.
await self._dispatch_to_top_intent(turn_context, intent, recognizer_result)
async def _dispatch_to_top_intent(
self, turn_context: TurnContext, intent, recognizer_result: RecognizerResult
):
if intent == "get-status":
await self._get_status(
turn_context, recognizer_result.properties["luisResult"]
)
elif intent == "None":
await self._none(
turn_context, recognizer_result.properties["luisResult"]
)
else:
await turn_context.send_activity(f"Dispatch unrecognized intent: {intent}.")
async def _get_status(self, turn_context: TurnContext, luis_result: LuisResult):
# await turn_context.send_activity(
# f"Matched intent {luis_result.top_scoring_intent}."
# )
#
# intents_list = "\n\n".join(
# [intent_obj.intent for intent_obj in luis_result.intents]
# )
# await turn_context.send_activity(
# f"Other intents detected: {intents_list}."
# )
#
outputs = []
if luis_result.entities:
for ent in luis_result.entities:
loc = self._AzMap.geocode(ent.entity, language='en-US')
cntry = loc.raw["address"]["country"]
out = self._filter_by_cntry(cntry)
if out is None:
cntry_code = loc.raw["address"]["countryCode"]
out = self._filter_by_cntry( cntry_code)
if out is not None:
confirmed, deaths, recovered, active = out
dt = helpers.to_human_readable(self._last_date)
outputs.append(f"As of {dt}, for Country: {cntry} there were {confirmed} confirmed cases, " +
f"{deaths} deaths, {recovered} recoveries and {active} active cases")
else:
#TODO: propose the card with options
outputs.append(f"Country : {cntry}, Code: {cntry_code} not found in the dataset, please try different spelling")
await turn_context.send_activity(
"\n".join(outputs)
)
async def _none(self, turn_context: TurnContext, luis_result: LuisResult):
await self._get_status(turn_context, luis_result)
return
def initialize(self):
self.geocoder = AzureMaps(
self.server.daconfig['azure maps']['primary key'])
def make_geocoder(cls, **kwargs):
return AzureMaps(env['AZURE_SUBSCRIPTION_KEY'], timeout=3, **kwargs)
class Geocode():
#SERVICES = []
#IGNORE = []
CURRENT_SERVICE = 0
geolocator_google = None
geolocator_here = None
geolocator_bing = None
geolocator_tomtom = None
geolocator_azure = None
geolocator_nominatum = None
#SHOW_ERRORS = True
def __init__(self, services=None, ignore=None):
self.SERVICES = services
self.IGNORE = ignore
############ SERVICES ############
def initOutput(self):
output = {}
output["formatted_address"] = None
output["latitude"] = None
output["longitude"] = None
output["accuracy"] = None
output["place_id"] = None
output["type"] = None
output["postcode"] = None
output["input_string"] = None
output["number_of_results"] = None
output["status"] = None
output["response"] = None
output["localidade"] = None
output["distrito"] = None
output["concelho"] = None
output["freguesia"] = None
output["service"] = self.SERVICES[self.CURRENT_SERVICE]['service']
return output
def google(self, addr, local, country, saveraw):
output = self.initOutput()
address = "" if addr is None else addr
address = address + ("" if local is None else "," + local)
address = address + ("" if country is None else "," + country)
# init service if not init yet
if not self.geolocator_google:
self.geolocator_google = GoogleV3(
api_key=self.SERVICES[self.CURRENT_SERVICE]['key'])
# geocode address
location = self.geolocator_google.geocode(
address, exactly_one=False) #, components={"country": "PT"})
if location is not None:
answer = location[0].raw
output['status'] = "OK"
output["formatted_address"] = location[0].address
output["latitude"] = location[0].latitude
output["longitude"] = location[0].longitude
output["accuracy"] = answer.get('geometry').get('location_type')
output["place_id"] = answer.get("place_id")
output["type"] = ",".join(answer.get('types'))
output["postcode"] = ",".join([
x['long_name'] for x in answer.get('address_components')
if 'postal_code' in x.get('types')
])
output["input_string"] = address
output["number_of_results"] = len(location)
output["localidade"] = ",".join([
x['long_name'] for x in answer.get('address_components')
if 'locality' in x.get('types')
]).split(',')[0]
output["service"] = self.SERVICES[self.CURRENT_SERVICE]['service']
if saveraw:
output["response"] = location[0].raw
else:
output['status'] = "ZERO_RESULTS"
return output
def tomtom(self, addr, local, country, saveraw):
output = self.initOutput()
# create query
address = "" if addr is None else addr
address = address + ("" if local is None else "," + local)
address = address + ("" if country is None else "," + country)
# init service if not init yet
if not self.geolocator_tomtom:
self.geolocator_tomtom = TomTom(api_key=self.SERVICES[
self.CURRENT_SERVICE]['key']) #,default_scheme = 'https')
# geocode address
location = self.geolocator_tomtom.geocode(
address, exactly_one=False) #, components={"country": "PT"})
if location is not None:
answer = location[0].raw
output['status'] = "OK"
output["latitude"] = location[0].latitude
output["longitude"] = location[0].longitude
output["accuracy"] = answer.get('score')
output["input_string"] = address
output["number_of_results"] = len(
location) #answer.get("numResults")
output["place_id"] = answer.get("id")
if answer.get("address"):
output["distrito"] = answer.get("address").get(
"countrySubdivision")
# maybe?
output["concelho"] = answer.get("address").get("municipality")
output["freguesia"] = answer.get("address").get(
"municipalitySubdivision")
output["formatted_address"] = answer.get('address').get(
'freeformAddress')
CPext = answer.get("address").get('extendedPostalCode')
CP = answer.get("address").get('postalCode')
if CPext:
CPext = CPext.split(',')[0]
CPext = CPext[:4] + '-' + CPext[4:]
output["postcode"] = CPext
elif CP:
output["postcode"] = CP.split(',')[0]
output["type"] = answer.get('type')
#output["query_type"] = answer.get("queryType")
# maybe?
#output["localidade"] = answer.get("address").get("municipality")
output["service"] = self.SERVICES[self.CURRENT_SERVICE]['service']
if saveraw:
output["response"] = location[0].raw
else:
output['status'] = "ZERO_RESULTS"
return output
def nominatim(self, addr, local, country, saveraw):
output = self.initOutput()
# create query
address = "" if addr is None else addr
address = address + ("" if local is None else "," + local)
address = address + ("" if country is None else "," + country)
'''
query = { 'street': data[1],
'city':data[2],
'country': 'Portugal'
}
'''
# init service if not init yet
if not self.geolocator_nominatum:
self.geolocator_nominatum = Nominatim(user_agent="tests_1")
# geocode address
location = self.geolocator_nominatum.geocode(address,
exactly_one=False,
addressdetails=True)
if location is not None:
answer = location[0].raw
output['status'] = "OK"
output["latitude"] = location[0].latitude
output["longitude"] = location[0].longitude
output["number_of_results"] = len(location)
#output["accuracy"] = answer.get('importance')
output["place_id"] = answer.get("osm_id")
output["input_string"] = address
if answer.get("address"):
output["postcode"] = re.sub(
'[^0-9-]+', '',
answer.get("address").get("postcode")) ###???
output["freguesia"] = answer.get("address").get("suburb")
output["localidade"] = answer.get("address").get("city")
if not output["localidade"]:
output["localidade"] = answer.get("address").get("town")
output["formatted_address"] = answer.get('address').get(
'display_name')
output["type"] = answer.get('osm_type')
output["service"] = self.SERVICES[self.CURRENT_SERVICE]['service']
if saveraw:
output["response"] = location[0].raw
else:
output['status'] = "ZERO_RESULTS"
return output
def bing(self, addr, local, country, saveraw):
output = self.initOutput()
# create query
address = "" if addr is None else addr
address = address + ("" if local is None else "," + local)
address = address + ("" if country is None else "," + country)
# init service if not init yet
if not self.geolocator_bing:
self.geolocator_bing = Bing(
api_key=self.SERVICES[self.CURRENT_SERVICE]['key'])
# geocode address
location = self.geolocator_bing.geocode(
address,
exactly_one=False) #culture='PT', include_neighborhood=True,
if location is not None:
answer = location[0].raw
output['status'] = "OK"
output["latitude"] = location[0].latitude
output["longitude"] = location[0].longitude
output["number_of_results"] = len(location)
if answer.get("address"):
output["formatted_address"] = answer.get('address').get(
'formattedAddress')
output["localidade"] = answer.get("address").get("locality")
output["distrito"] = answer.get("address").get("adminDistrict")
output["concelho"] = answer.get("address").get(
"adminDistrict2")
output["freguesia"] = answer.get("address").get("neighborhood")
output["postcode"] = answer.get("address").get("postalCode")
output["accuracy"] = answer.get('confidence')
output["input_string"] = address
output["service"] = self.SERVICES[self.CURRENT_SERVICE]['service']
if saveraw:
output["response"] = location[0].raw
else:
output['status'] = "ZERO_RESULTS"
return output
def here(self, addr, local, country, saveraw):
output = self.initOutput()
# create query
address = "" if addr is None else addr
address = address + ("" if local is None else "," + local)
address = address + ("" if country is None else "," + country)
# init service if not init yet
if not self.geolocator_here:
self.geolocator_here = Here(
app_id=self.SERVICES[self.CURRENT_SERVICE]['app_id'],
app_code=self.SERVICES[self.CURRENT_SERVICE]['app_code'])
# geocode address
location = self.geolocator_here.geocode(address,
exactly_one=False,
language="pt-PT")
if location is not None:
answer = location[0].raw
output['status'] = "OK"
output["latitude"] = location[0].latitude
output["longitude"] = location[0].longitude
output["number_of_results"] = len(location)
output["input_string"] = address
output["accuracy"] = answer.get('Relevance')
if answer.get("Location"):
output["formatted_address"] = answer.get("Location").get(
'Address').get('Label')
output["place_id"] = answer.get("Location").get("LocationId")
if answer.get("Location"):
if answer.get("Location").get("Address"):
output["postcode"] = answer.get("Location").get(
"Address").get("PostalCode")
# all 4 are not tghrustworthy
output["freguesia"] = answer.get("Location").get(
"Address").get("District")
output["distrito"] = answer.get("Location").get(
"Address").get("County")
output["concelho"] = answer.get("Location").get(
"Address").get("City")
output["localidade"] = answer.get("Location").get(
"Address").get("City")
output["service"] = self.SERVICES[self.CURRENT_SERVICE]['service']
if saveraw:
output["response"] = location[0].raw
else:
output['status'] = "ZERO_RESULTS"
return output
###
def azure(self, addr, local, country, saveraw):
output = self.initOutput()
# create query
address = "" if addr is None else addr
address = address + ("" if local is None else "," + local)
address = address + ("" if country is None else "," + country)
# init service if not init yet
if not self.geolocator_azure:
self.geolocator_azure = AzureMaps(
subscription_key=self.SERVICES[self.CURRENT_SERVICE]['key'])
# geocode address
location = self.geolocator_azure.geocode(address,
exactly_one=False,
language="pt-PT")
if location is not None:
answer = location[0].raw
output['status'] = "OK"
output["latitude"] = location[0].latitude
output["longitude"] = location[0].longitude
output["number_of_results"] = len(location)
output["input_string"] = address
output["accuracy"] = answer.get('score')
output["place_id"] = answer.get("id")
if answer.get("address"):
output["formatted_address"] = answer.get('address').get(
'freeformAddress')
output["distrito"] = answer.get("address").get(
"countrySubdivision")
# maybe?
output["concelho"] = answer.get("address").get("municipality")
output["freguesia"] = answer.get("address").get(
"municipalitySubdivision")
CPext = answer.get("address").get('extendedPostalCode')
CP = answer.get("address").get('postalCode')
if CPext:
CPext = CPext.split(',')[0]
CPext = CPext[:4] + '-' + CPext[4:]
output["postcode"] = CPext
elif CP:
output["postcode"] = CP.split(',')[0]
output["type"] = answer.get('type')
output["service"] = self.SERVICES[self.CURRENT_SERVICE]['service']
if saveraw:
output["response"] = location[0].raw
else:
output['status'] = "ZERO_RESULTS"
return output
############ PROCESS FILE ############
def getService(self):
if self.CURRENT_SERVICE >= len(self.SERVICES):
raise UnableToGeocode("Unable to geocode entity.")
if len(self.IGNORE) >= len(self.SERVICES):
raise OutOfServices("No service available.")
for i in self.SERVICES:
if self.SERVICES[self.CURRENT_SERVICE]['service'] in self.IGNORE:
self.CURRENT_SERVICE = self.CURRENT_SERVICE + 1
if self.CURRENT_SERVICE >= len(self.SERVICES):
raise UnableToGeocode("Unable to geocode entity.")
else:
break
if "GOOGLE" in self.SERVICES[self.CURRENT_SERVICE]['service']:
return self.google
elif "TOMTOM" in self.SERVICES[self.CURRENT_SERVICE]['service']:
return self.tomtom
elif "NOMINATUM" in self.SERVICES[self.CURRENT_SERVICE]['service']:
return self.nominatim
elif "BING" in self.SERVICES[self.CURRENT_SERVICE]['service']:
return self.bing
elif "HERE" in self.SERVICES[self.CURRENT_SERVICE]['service']:
return self.here
elif "AZURE" in self.SERVICES[self.CURRENT_SERVICE]['service']:
return self.azure
return None
# service = None => all available
def geocode(self,
addr=None,
local=None,
country="Portugal",
saveraw=True,
service=None):
geocoded = False
self.CURRENT_SERVICE = 0
geocode_result = None
if service:
for s in self.SERVICES:
if s['service'] != service:
self.IGNORE.append(s['service'])
while not geocoded:
try:
serv = self.getService()
geocode_result = serv(addr, local, country, saveraw)
if geocode_result['status'] == "OK":
geocoded = True
break
else:
self.CURRENT_SERVICE = self.CURRENT_SERVICE + 1
'''
else:
if DEBUG:
logger.error ('\n--------------------------------------------------------------------')
logger.error ('ERROR: no addr/name for id_localization [{}].'.format(address.split('|')[0]))
logger.error ('Passing to next address.')
logger.error ('--------------------------------------------------------------------')
CURRENT_SERVICE = 0
geocode_result = initOutput()
geocode_result['id_localization'] = address.split('|')[0]
geocode_result['status'] = "NO_DATA"
break
'''
except UnableToGeocode as e:
if self.SHOW_ERRORS:
pass
#logger.error ('\n--------------------------------------------------------------------')
#logger.error ('ERROR: Unable to geocode addr [{}].'.format(addr))
#logger.error ('Passing to next address.')
#logger.error ('--------------------------------------------------------------------')
self.CURRENT_SERVICE = 0
geocode_result = self.initOutput()
geocode_result['status'] = "UNABLE"
geocode_result['service'] = "ALL"
break
except OutOfServices as e:
#if self.SHOW_ERRORS:
# logger.error ('\n--------------------------------------------------------------------')
# logger.error ('ERROR: you reached the limit on all services. No more services available.')
# logger.error ('Saving the all sucessuful results and exiting the application.')
# logger.error ('--------------------------------------------------------------------')
raise
#return None
except (GeocoderQueryError, GeocoderAuthenticationFailure,
GeocoderInsufficientPrivileges, ConfigurationError):
#if self.SHOW_ERRORS:
# logger.error ('\n--------------------------------------------------------------------')
# logger.error ('ERROR: something wrong with either the service or the query.')
# logger.error ('Check service: [{}]'.format(self.SERVICES[self.CURRENT_SERVICE]['id']))
# logger.error ('Passing to the next service.')
# logger.error ('--------------------------------------------------------------------')
self.IGNORE.append(
self.SERVICES[self.CURRENT_SERVICE]['service'])
except GeocoderQuotaExceeded:
#if self.SHOW_ERRORS:
# logger.error ('\n--------------------------------------------------------------------')
# logger.error ('ERROR: you have reached the end of your quota for service [{}].'.format(self.SERVICES[self.CURRENT_SERVICE]['id']))
# logger.error ('Passing to the next service.')
# logger.error ('--------------------------------------------------------------------')
self.IGNORE.append(
self.SERVICES[self.CURRENT_SERVICE]['service'])
except GeocoderTimedOut:
#if self.SHOW_ERRORS:
# logger.error ('\n--------------------------------------------------------------------')
# logger.error ('TIMEOUT: something went wrong with the geocoding the address: [{}].'.format(addr))
# logger.error ('while using service [{}].'.format(self.SERVICES[self.CURRENT_SERVICE]['id']))
# logger.error ('Passing to the next service.')
# logger.error ('--------------------------------------------------------------------')
self.IGNORE.append(
self.SERVICES[self.CURRENT_SERVICE]['service'])
except (GeocoderServiceError, GeocoderUnavailable):
#if self.SHOW_ERRORS:
# logger.error ('\n--------------------------------------------------------------------')
# logger.error ('ERROR: service unavailable or unknown error for service [{}].'.format(self.SERVICES[self.CURRENT_SERVICE]['id']))
# logger.error ('Passing to the next service.')
# logger.error ('--------------------------------------------------------------------')
self.IGNORE.append(
self.SERVICES[self.CURRENT_SERVICE]['service'])
except GeocoderNotFound:
#if self.SHOW_ERRORS:
# logger.error ('\n--------------------------------------------------------------------')
# logger.error ('ERROR: unknown service > [{}].'.format(self.SERVICES[self.CURRENT_SERVICE]['id']))
# logger.error ('check if this service still exists!')
# logger.error ('Passing to the next service.')
# logger.error ('--------------------------------------------------------------------')
self.IGNORE.append(
self.SERVICES[self.CURRENT_SERVICE]['service'])
except Exception as e:
#logger.error ('\n--------------------------------------------------------------------')
#logger.error("Unknown catastrophic error while processing address: {}".format(addr))
#logger.error('while using service > [{}].'.format(self.SERVICES[self.CURRENT_SERVICE]['id']))
#logger.error("Check the error and correct it before restart the application.")
#logger.error(str(e))
#logger.error('--------------------------------------------------------------------')
raise
#return None
return geocode_result
tout = 5
geodata = OrderedDict()
USE_GEOPY = False
for i, q in enumerate(query_lst):
info_d = OrderedDict()
if 'county' in q:
place = q.split(' county, ')[0] + ' county'
else:
place = q.split(', ')[0]
if USE_GEOPY:
g = AzureMaps(subscription_key=AZK, domain='atlas.microsoft.com')
location = g.geocode(q)
#r = g.geocode('New York county, NY, USA')
#r.raw['viewport']
else:
url_Azure = 'https://atlas.microsoft.com/search/address/json'
params = {
'subscription-key': AZK,
'api-version': 1.0,
'query': q,
'typeahead': False,
'limit': 1
}
r = requests.get(url_Azure, params=params)
location = r.json()['results']
def initialize(self):
from geopy.geocoders import AzureMaps
self.geocoder = AzureMaps(
self.server.daconfig['azure maps']['primary key'])
class HeroBot(ActivityHandler):
def __init__(self, config: DefaultConfig):
luis_application = LuisApplication(
config.LUIS_APP_ID,
config.LUIS_API_KEY,
"https://" + config.LUIS_API_HOST_NAME,
)
luis_options = LuisPredictionOptions(
include_all_intents=True, include_instance_data=True
)
self.recognizer = LuisRecognizer(luis_application, luis_options, True)
self.fetch_dataset()
self._AzMap = AzureMaps(subscription_key=config.AZURE_MAPS_KEY)
def fetch_dataset(self):
self._confirmed = pd.read_csv(
"https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_19-covid-Confirmed.csv",
index_col=["Country/Region", "Province/State"]).iloc[:, -1]
self._deaths = pd.read_csv(
"https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_19-covid-Deaths.csv",
index_col=["Country/Region", "Province/State"]).iloc[:, -1]
self._recovered = pd.read_csv(
"https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_19-covid-Recovered.csv",
index_col=["Country/Region", "Province/State"]).iloc[:, -1]
self._curr_date = pd.to_datetime(self._confirmed.name)
def _filter_by_cntry(self, cntry):
out = None
try:
out = (self._confirmed[cntry].sum(), self._deaths[cntry].sum(), self._recovered[cntry].sum())
except Exception as e:
out = None
print(f"[WARNING] Encountered country matching problem, Country = {e}")
return out
async def on_members_added_activity(
self, members_added: [ChannelAccount], turn_context: TurnContext
):
for member in members_added:
if member.id != turn_context.activity.recipient.id:
card = HeroCard(
title="Welcome to the COVID-19 Information bot",
images=[
CardImage(
url="https://i.imgur.com/zm095AG.png"
)
],
buttons=[
CardAction(
type=ActionTypes.open_url,
title="Repository link",
value="https://github.com/vykhand/realherobot",
)
],
)
repl = MessageFactory.list([])
repl.attachments.append(CardFactory.hero_card(card))
await turn_context.send_activity(repl)
async def on_message_activity(self, turn_context: TurnContext):
# First, we use the dispatch model to determine which cognitive service (LUIS or QnA) to use.
recognizer_result = await self.recognizer.recognize(turn_context)
# Top intent tell us which cognitive service to use.
intent = LuisRecognizer.top_intent(recognizer_result)
# Next, we call the dispatcher with the top intent.
await self._dispatch_to_top_intent(turn_context, intent, recognizer_result)
async def _dispatch_to_top_intent(
self, turn_context: TurnContext, intent, recognizer_result: RecognizerResult
):
if intent == "get-status":
await self._get_status(
turn_context, recognizer_result.properties["luisResult"]
)
elif intent == "None":
await self._none(
turn_context, recognizer_result.properties["luisResult"]
)
else:
await turn_context.send_activity(f"Dispatch unrecognized intent: {intent}.")
async def _get_status(self, turn_context: TurnContext, luis_result: LuisResult):
# await turn_context.send_activity(
# f"Matched intent {luis_result.top_scoring_intent}."
# )
#
# intents_list = "\n\n".join(
# [intent_obj.intent for intent_obj in luis_result.intents]
# )
# await turn_context.send_activity(
# f"Other intents detected: {intents_list}."
# )
#
outputs = []
if luis_result.entities:
for ent in luis_result.entities:
loc = self._AzMap.geocode(ent.entity, language='en-US')
cntry = loc.raw["address"]["country"]
out = self._filter_by_cntry(cntry)
if out is None:
cntry_code = loc.raw["address"]["countryCode"]
out = self._filter_by_cntry( cntry_code)
if out is not None:
confirmed, deaths, recovered = out
dt = helpers.to_human_readable(self._curr_date)
outputs.append(f"As of {dt}, for Country: {cntry} there were {confirmed} confirmed cases, {deaths} deaths and {recovered} recoveries")
else:
#TODO: propose the card with options
outputs.append(f"Country : {cntry}, Code: {cntry_code} not found in the dataset, please try different spelling")
await turn_context.send_activity(
"\n".join(outputs)
)
async def _none(self, turn_context: TurnContext, luis_result: LuisResult):
await self._get_status(turn_context, luis_result)
return
def get_geodata(geocoder_to_use, query_list, use_local=True, alt_prefix=''):
"""
Wrapper function for using one of four geocoders: 'Nominatim', 'GoogleV3',
'ArcGis', 'AzureMaps', to retrieve the geographical data of places in
query_list.
Parameters
----------
:param: geocoder_to_use (str): to switch geocoding service
:param: query_list (list): a list of cities, places or counties
:param: use_local (bool), default=True: a local file returned if found
:param: alt_prefix (str), default='': to retrieve a geojson file tagged
with that prefix => use_local=True (old file).
Returns
-------
geodata (odict): odict_keys(['loc', 'box']) where
loc=['lat', 'lon'] and box=[[NE lat, lon], [SW lat, lon]].
Note: the loc key identifies the place queried, not the entire string.
Example
-------
>>> geo_Nom = get_geodata('Nominatim', ['New York City, NY, USA'])
>>> geo_Nom['New York City'].keys()
odict_keys(['loc', 'box'])
"""
# init checks:
if geocoder_to_use not in geocs:
msg = 'Function setup for these geocoders: {}, not for: {}'
msg = msg.format(geocs, geocoder_to_use)
raise Exception(msg)
if (not isinstance(query_list, list)):
msg = '"query_list" must be a list. Given is: {}'
msg = msg.format(type(query_list))
return TypeError(msg)
# to check for & save local file; base name w/o extension:
out = 'geodata_' + geocoder_to_use[:3]
no_fetching = len(alt_prefix)
if no_fetching:
# implied: pre-existing file: no over writing.
use_local = True
if alt_prefix[-1] != '_':
out = alt_prefix + '_' + out
else:
out = alt_prefix + out
if use_local:
geodata = gc4utils.get_geo_file(
os.path.join(gc4settings.DIR_GEO, out + '.json'))
if isinstance(geodata, dict):
return geodata
else:
if no_fetching:
print("Non overwritable file not found:\t{}".format(out))
return None
else:
# Fetch it
use_local = False
if not use_local:
tout = 5
idx = geocs.index(geocoder_to_use)
if idx == 0:
g = Nominatim(user_agent='this_app',
country_bias='USA',
timeout=tout)
elif idx == 1:
g = GoogleV3(api_key=GOOGLE_KEY, timeout=tout)
elif idx == 2:
g = ArcGIS(user_agent='this_app', timeout=tout)
else:
# original setup stopped working 9/12/18: unable to resolve the
# http 400 error; reverted to request/json.
g = AzureMaps(subscription_key=AZURE_KEY,
user_agent='ths_app',
timeout=tout)
#url_Azure = 'https://atlas.microsoft.com/search/address/json'
geodata = OrderedDict()
for i, q in enumerate(query_list):
info_d = OrderedDict()
if 'county' in q:
place = q.split(' county, ')[0] + ' county'
else:
place = q.split(', ')[0]
if idx == 0:
location = g.geocode(
q, addressdetails=True).raw #exactly_one=False,
else:
location = g.geocode(q).raw
if isinstance(location, list):
location = location[0]
if len(location): # not sure that's a sufficient check...
if idx == 0:
# pt location
info_d['loc'] = [
float(location['lat']),
float(location['lon'])
]
# bounding boxes as 2 corner pts: [NE], [SW]
info_d['box'] = [[
float(location['boundingbox'][1]),
float(location['boundingbox'][3])
],
[
float(location['boundingbox'][0]),
float(location['boundingbox'][2])
]]
elif idx == 1:
info_d['loc'] = [
location['geometry']['location']['lat'],
location['geometry']['location']['lng']
]
info_d['box'] = [[
location['geometry']['viewport']['northeast']['lat'],
location['geometry']['viewport']['northeast']['lng']
],
[
location['geometry']['viewport']
['southwest']['lat'],
location['geometry']['viewport']
['southwest']['lng']
]]
elif idx == 2:
info_d['loc'] = [
location['location']['y'], location['location']['x']
]
info_d['box'] = [[
location['extent']['ymax'], location['extent']['xmax']
], [
location['extent']['ymin'], location['extent']['xmin']
]]
else:
info_d['loc'] = [
location['position']['lat'],
location['position']['lon']
]
info_d['box'] = [
[
location['viewport']['topLeftPoint']['lat'],
location['viewport']['btmRightPoint']['lon']
],
[
location['viewport']['btmRightPoint']['lat'],
location['viewport']['topLeftPoint']['lon']
]
]
geodata[place] = info_d
# save file (overwrite=default)
outfile = os.path.join(DIR_GEO, out)
gc4utils.save_file(outfile, 'json', geodata)
return geodata