Пример #1
0
class GeoCoder:
    def __init__(self):
        # use to get location from lat, log coordinates. (https://opencagedata.com/tutorials/geocode-in-python)
        try:
            root_path = os.path.dirname(os.path.abspath(__file__))
            with open(file=os.path.join(root_path, "GeoKey.txt"),
                      mode="r") as f:
                key = f.read()
                self.__geocoder = OpenCageGeocode(key)
        except FileNotFoundError:
            app_logger.warning("File GeoKey.txt not found")
            raise AppException("File contain Key for geocoder not found.")

    def get_location(self, lat: float, long: float):
        results = self.__geocoder.reverse_geocode(lat, long)
        state = ""
        country = ""
        if len(results) > 0:
            component = results[0].get("components")
            if component:
                country = component.get("country")
                state = component.get("state")

        return Location(latitude=lat,
                        longitude=long,
                        province=state,
                        country=country)
    def verificarCoordenadas(
        self
    ):  #Verifica se as coordenadas da ocorrência correspondem ao estado indicado.
        key = '5b7a26bfcd904e07a49f085037f6d7ba'
        geocoder = OpenCageGeocode(key)

        if self.data is None:
            self.listdata()
        count = 0
        for linha in self.data:
            try:
                results = geocoder.reverse_geocode(float(linha[29]),
                                                   float(linha[30]),
                                                   no_annotations='1',
                                                   language='pt')
                if results and len(results):
                    count += 1
                    coord = (results[0]['components']['state_code'])
                    #print (coord)
                    #if coord == linha[26]:
                    #   print ("A coordenada corresponde ao estado.")
                if coord != linha[26]:
                    print("A coordenada da ocorrência", count + 1,
                          "não correspode ao estado da ocorrência.")
                    print("Estado indicado: ", linha[26],
                          "|  Estado indicado pela coordenada: ", coord)

            except InvalidInputError as ex:
                print(ex)
Пример #3
0
    def command(self, fromcall, message, ack):
        api_key = self.config["services"]["aprs.fi"]["apiKey"]
        try:
            aprs_data = plugin_utils.get_aprs_fi(api_key, fromcall)
        except Exception as ex:
            LOG.error("Failed to fetch aprs.fi data {}".format(ex))
            return "Failed to fetch location"

        # LOG.debug("LocationPlugin: aprs_data = {}".format(aprs_data))
        lat = aprs_data["entries"][0]["lat"]
        lon = aprs_data["entries"][0]["lng"]

        try:
            utils.check_config_option(self.config, "opencagedata", "apiKey")
        except Exception as ex:
            LOG.error("Failed to find config opencage:apiKey {}".format(ex))
            return "No opencage apiKey found"

        try:
            opencage_key = self.config["opencagedata"]["apiKey"]
            geocoder = OpenCageGeocode(opencage_key)
            results = geocoder.reverse_geocode(lat, lon)
        except Exception as ex:
            LOG.error("Couldn't fetch opencagedata api '{}'".format(ex))
            # Default to UTC instead
            localzone = pytz.timezone("UTC")
        else:
            tzone = results[0]["annotations"]["timezone"]["name"]
            localzone = pytz.timezone(tzone)

        return self.build_date_str(localzone)
Пример #4
0
    def save(self, *args, **kwargs):
        initial_start = self.start
        self.slug = slugify(self.name)
        super().save(*args, **kwargs)

        # Check if the start location has changed
        start_location_changed = False

        if not self.start:
            self.start = self.calculate_walk_start()
            start_location_changed = True

        if self.start != initial_start:
            start_location_changed = True

        # check if the cache_has expired
        if self.reverse_geocode_cache_time is None:
            cache_expired = True
        else:
            cache_expired = self.reverse_geocode_cache_time < (
                now() - timedelta(days=180)
            )

        # update the location details if we need to
        if start_location_changed or cache_expired:
            if settings.OPENCAGE_API_KEY:
                geocoder = OpenCageGeocode(settings.OPENCAGE_API_KEY)

                try:
                    results = geocoder.reverse_geocode(
                        round(self.start[1], 6),
                        round(self.start[0], 6),
                        language="en",
                        limit=1,
                    )

                    if results and len(results):
                        self.what3words = (
                            results[0].get("annotations").get("what3words").get("words")
                        )
                        self.geohash = results[0].get("annotations").get("geohash")
                        self.continent = results[0].get("components").get("continent")
                        self.country = results[0].get("components").get("country")
                        self.state = results[0].get("components").get("state")
                        self.county = results[0].get("components").get("county")
                        self.city = results[0].get("components").get("city")
                        self.suburb = results[0].get("components").get("suburb")
                        self.road = results[0].get("components").get("road")
                        self.postcode = results[0].get("components").get("postcode")
                        self.formatted = results[0].get("formatted")

                        self.reverse_geocode_cache_time = now()
                        super().save(*args, **kwargs)

                except RateLimitExceededError as e:
                    print(e)

        self.route.transform(3857)
        self.route_length = D(m=self.route.length).mi
        super().save(*args, **kwargs)
def coordinates_to_address(latitude, longitude):

    key = '4745929c7204418da8d7bd0b0b693a69'
    geocoder = OpenCageGeocode(key)

    results = geocoder.reverse_geocode(latitude, longitude)
    # pprint(results)
    return results
Пример #6
0
def get_location(lat='X', lng='X'):
    key = '2ead87ca7a984517ba94f2b73a70edc5'
    file_gps = "hw16.txt"
    listGps = []
    geocoder = OpenCageGeocode(key)
    if lat != 'X' and lng != 'X':
        listGps.append(str(lat + ', ' + lng))
        results = geocoder.reverse_geocode(lat, lng)
        print_location(results)
    else:
        with open(file_gps,'r') as file:
            listGps = file.readlines()
        for i in listGps:
            try:
                latlng = i.split(', ')
                results = geocoder.reverse_geocode(latlng[0], latlng[1])
            except:
                print("Не верно введённые данные")
                break
            print_location(results)
Пример #7
0
    def validar_localidade(self, lat, long, estado):
        geocode = OpenCageGeocode(self.chave)
        resultado = geocode.reverse_geocode(lat, long)

        if 'components' in resultado[0] and 'state' in resultado[0]['components'] and 'state_code' in resultado[0]['components']:
            if resultado[0]['components']['state'] == estado or resultado[0]['components']['state_code'] == estado :
                return True
            else:
                return False
        
        return False
 def verify_lat_long(self, city, lat, long):
     key = '65b25c705a5349ad99c824ca809363b7'
     geocoder = OpenCageGeocode(key)
     results = geocoder.reverse_geocode(lat, long)
     try:
         r_city = results[0]['components']['city']
     except Exception as identifier:
         try:
             r_city = results[0]['components']['island']
         except Exception as identifier:
             r_city = ""
     return r_city == city
Пример #9
0
def reverse_geocode(latitude, longitude):
    KEY = GEO_KEY
    geocoder = OpenCageGeocode(KEY)
    try:
        results = geocoder.reverse_geocode(latitude, longitude)
        region = results[0]['components']['state']
        city = results[0]['formatted']
        address = (str(region) + ',' + str(city))
        # print(address)
    except Exception:
        print("feilure")
    else:
        return address
Пример #10
0
    def process(self, packet):
        fromcall = packet.get("from")
        message = packet.get("message_text", None)
        # ack = packet.get("msgNo", "0")

        api_key = self.config["services"]["aprs.fi"]["apiKey"]

        # optional second argument is a callsign to search
        a = re.search(r"^.*\s+(.*)", message)
        if a is not None:
            searchcall = a.group(1)
            searchcall = searchcall.upper()
        else:
            # if no second argument, search for calling station
            searchcall = fromcall

        try:
            aprs_data = plugin_utils.get_aprs_fi(api_key, searchcall)
        except Exception as ex:
            LOG.error(f"Failed to fetch aprs.fi data {ex}")
            return "Failed to fetch location"

        # LOG.debug("LocationPlugin: aprs_data = {}".format(aprs_data))
        if not len(aprs_data["entries"]):
            LOG.error("Didn't get any entries from aprs.fi")
            return "Failed to fetch aprs.fi location"

        lat = aprs_data["entries"][0]["lat"]
        lon = aprs_data["entries"][0]["lng"]

        try:
            self.config.exists("opencagedata.apiKey")
        except Exception as ex:
            LOG.error(f"Failed to find config opencage:apiKey {ex}")
            return "No opencage apiKey found"

        try:
            opencage_key = self.config["opencagedata"]["apiKey"]
            geocoder = OpenCageGeocode(opencage_key)
            results = geocoder.reverse_geocode(lat, lon)
        except Exception as ex:
            LOG.error(f"Couldn't fetch opencagedata api '{ex}'")
            # Default to UTC instead
            localzone = pytz.timezone("UTC")
        else:
            tzone = results[0]["annotations"]["timezone"]["name"]
            localzone = pytz.timezone(tzone)

        return self.build_date_str(localzone)
Пример #11
0
    async def get(self, lat, lon):

        # TODO use geocode to validate lat, lon
        geocoder = OpenCageGeocode(OPEN_CAGE_API_KEY)

        try:
            results = geocoder.reverse_geocode(lat, lon)
            address = results[0]['formatted']
        except Exception as ex:
            raise tornado.web.HTTPError(404)

        data = await self.get_weather(lat, lon)
        data = json.loads(data.decode('utf8'))
        data['address'] = address
        data = parser(data)
        self.write(data)
Пример #12
0
    def get(self):
        query = self.request.get('q')
        key = OPEN_CAGE_KEY

        geocoder = OpenCageGeocode(key)

        if not query:
            latlong = self.request.headers.get('X-AppEngine-CityLatLong')
            latlong = string.split(latlong, ',')

            result = geocoder.reverse_geocode(latlong[0], latlong[1])
        else:
            result = geocoder.geocode(query, limit=4, language='fr')

        self.response.headers['Access-Control-Allow-Origin'] = ACCESS_CONTROL_ALLOW_ORIGIN_URL
        self.response.headers['Content-Type'] = 'application/json; charset=utf-8'
        self.response.write(json.dumps(result))
Пример #13
0
def IssLocation(request):
    # API request to open-notify for lat/long of ISS
    response = requests.get("http://api.open-notify.org/iss-now.json")
    data = response.json()
    location = data['iss_position']
    
    # Reverse_geocodes the lat/long using opencage API
    key = '3a3f13fb513541f79345ff8a0eebbe1d'
    geocoder = OpenCageGeocode(key)
    reverse_geocode = geocoder.reverse_geocode(location['latitude'], location['longitude'])
    
    # Ocassionally goeocoding doesn't work, hence error handling
    try:
        result = reverse_geocode[0]['formatted']
    except IndexError:
        result = "It's in space. Duh!"
    
    return render(request, 'stations/isslocation.html', {'result': result})
Пример #14
0
def request_location_from_coords(lat, lon):
    key = '95d6bcc151474d98a00b4181e587bf15'
    geocoder = OpenCageGeocode(key)
    response = geocoder.reverse_geocode(lat, lon)
    info = response[0]['components']
    all_info = []
    if 'city' in info:
        all_info.append(info['city'])
    if 'state' in info:
        all_info.append(info['state'])
    if 'country' in info:
        all_info.append(info['country'])
    if 'body_of_water' in info:
        all_info.append("The " + info['body_of_water'])
    final_string = ""
    for item in all_info:
        final_string += item + ", "
    return final_string.rstrip(', ')
Пример #15
0
    def sacarUbicacion(self):
        key = 'd10cf63d329b4c158a1cd23baf90dc4a'
        geocoder = OpenCageGeocode(key)
        # Latitude & Longitude input
        Latitude = self.convertirGrados2Decimal(self.LatGrados,
                                                self.LatMinutos,
                                                self.LatSegundos)
        Longitude = self.convertirGrados2Decimal(self.LonGrados,
                                                 self.LonMinutos,
                                                 self.LonSegundos)
        try:
            results = geocoder.reverse_geocode(Latitude,
                                               Longitude,
                                               language='es',
                                               no_annotations='1')
            dicComp = {}
            dicComp = results[0]['components']
            if dicComp and len(dicComp):
                try:
                    self.elPueblo = dicComp['town']
                except:
                    try:
                        self.elPueblo = dicComp['municipality']
                    except:
                        try:
                            self.elPueblo = dicComp['village']
                        except:
                            try:
                                self.elPueblo = dicComp['city']
                            except:
                                self.elPueblo = 'Sin Datos'

        except RateLimitExceededError as ex:
            print(ex)
            # Your rate limit has expired. It will reset to 2500 next day
        except InvalidInputError as ex:
            # this happens for example with invalid unicode in the input data
            print(ex)
def get_district_coordinates(lat, long):
    # Lấy giá trị từ lat long giới hạn 2500/ngày api
    geocoder = OpenCageGeocode(key)
    results = geocoder.reverse_geocode(lat, long)
    data_district = ""
    for key_dis in key_district:
        try:
            data_district = results[0]['components'][key_dis]
            break
        except:
            print("[X] error : !!!!!!!! :" + key_dis +" + (" + str(lat) +","+ str(long)+")")
            continue

    # Tiền xử lý dữ liệu trước
    data_district = str(data_district).lower()
    for str_rm in str_remove:
        data_district = data_district.replace(str_rm, '')
    data_district    = re.sub(' +', ' ', data_district)
    for char_replace in replace_str:
        data_district = data_district.replace(char_replace, replace_str[char_replace])

    # Xử lý lỗi từ API
    for dis_replace in replace_miss:
        data_district = data_district.replace(dis_replace, replace_miss[dis_replace])

    data_district = data_district.strip()
    data_district_re = data_district.replace(" ", "")

    # Lấy giá trị trong map
    print(data_district + "++++" + data_district_re)
    district_name = None
    for district, value in cf.district_convert.items():
        if (data_district == district.lower()) or (data_district_re == value.lower()):
            district_name = value
            break
    return district_name
Пример #17
0
    def checkLatLong(self):
        self.latLong = [
        ]  # Lista de listas: cada sublista esta no formato [lat,long]
        self.city = []  # Lista de cidades
        self.indexNotOK = []
        [
            self.latLong.append([i.split(";")[29],
                                 i.split(";")[30]]) for i in self.arquivo[1:60]
        ]
        [self.city.append(i.split(";")[27]) for i in self.arquivo[1:60]]

        key = 'b3581bc610a644f896bd87fc2cd8e6ce'
        geocoder = OpenCageGeocode(key)

        for i in range(0, len(self.latLong)):
            results = geocoder.reverse_geocode(self.latLong[i][0],
                                               self.latLong[i][1])
            if self.city[i].lower() != results[0]['components']['city'].lower(
            ):
                #self.notOK.append([["City from geocoder: "+results[0]['components']['city']],["City from csv-file: "+self.city[i]],["Index: "+str(i)]])
                self.indexNotOK.append(i)

        print("Indices dos dados conflitantes:", self.indexNotOK)
        return self.indexNotOK
Пример #18
0
    def verifica_lat_long(self):

        #chave necessária para acessar o pacote geocode
        key = '3f1d63102a214341bc1a293171077d33'
        #inicia o pacote com a chave
        geocoder = OpenCageGeocode(key)

        #para cada linha dos dados teste
        for i in range(len(self.data)):
            #observa latitude e longitude informada
            latitude = (self.data[i][29])
            longitude = (self.data[i][30])
            #se não for o primeiro campo (informações sobre campos)
            if (i > 0):
                #Recebe informações sobre latitude e longitude
                results = geocoder.reverse_geocode(latitude, longitude)
                #Se dentre os resultados, houver componentes carregados sobre localização
                if (len(results) >= 0 and results != []):
                    #Se tiver categoria dentre os componentes do resultado
                    if ('_category' in results[0]['components']):
                        #Se a categoria para o resultado for do tipo "place"
                        if (results[0]['components']['_category'] == 'place'):
                            #Caso exista cidade nos componentes
                            if ('town' in results[0]['components']):

                                #retira pontuação para comparação com base de dados (base de dados sem pontuação)
                                cidade_coord = ''.join(
                                    (c for c in unicodedata.normalize(
                                        'NFD', (
                                            results[0]['components']['town']))
                                     if unicodedata.category(c) != 'Mn'))
                                #Se cidade informada na base de dados for igual a cidade observada a partir de latitude e longitude
                                if (self.data[i][27] == cidade_coord):
                                    print("Igual")
                                else:
                                    print(
                                        "Localização geográfica não correspondente ao informado \n\n"
                                    )
                                    print("Localização informada:" +
                                          self.data[i][27] + "\n\n")
                                    print(
                                        "Localização informada por coordenadas: "
                                        + cidade_coord)

                        #Se a categoria for do tipo natural/water
                        elif (results[0]['components']['_category'] ==
                              'natural/water'):
                            if ('body_of_water' in results[0]['components']):

                                #Printa o tipo do corpo de água
                                print("Localização ocêanica:" + str(
                                    results[0]['components']['body_of_water']))
                                print("Localização por coordenadas" +
                                      str(results[0]['geometry']) + "\n")

                        #Caso a categoria não for dos tipos "place" ou "natural/water"
                        else:

                            #Imprime o tipo da localização
                            print("Localização do tipo: " +
                                  str(results[0]['components']['_category']))
                            tipo = str(results[0]['components']['_type'])
                            #Imprime a localização pelo seu tipo
                            if (tipo in results[0]['components']):
                                print(str(results[0]['components'][tipo]))
                                print("Localização por coordenadas" +
                                      str(results[0]['geometry']) + "\n")
Пример #19
0
#!/usr/bin/env python
# -*- coding: UTF-8 -*-

print "Content-Type: text/plain;charset=utf-8"
print

import sys
import opencage
from opencage.geocoder import OpenCageGeocode
import json
import cgi, cgitb

# Create instance of FieldStorage
data = cgi.FieldStorage()

# Get data from fields
lat = data["lat"].value
lng = data["lng"].value

#print lat
#print lng

key = '2fcab5a7b512e6cabb28f107802aa0d0'
geocoder = OpenCageGeocode(key)
results = geocoder.reverse_geocode(lat, lng)

print json.dumps(results)
class OpenCageAddressService(AddressService):
    """
    Class represents Open Cage implementation of AddressService class. It 
    provides reverse and forward geocoding. 

    Open cage is rate limited based on API access tier, so request 
    response time will be variable. 
    """
    def __init__(self):
        self.client = None
        self.num_addresses_processed = 0

    def load_config(self, config_file):
        """Resonsible for loading configs and setting up client"""
        config = configparser.ConfigParser()
        config.read(config_file)
        auth_key = config.get('OPEN CAGE', 'auth_key')
        self.client = OpenCageGeocode(auth_key)

    def send_request(self, params, address_data):
        """Responsible for sending a request to service"""
        try:
            # forward geocoding
            if int(params["options"]) == 2:
                address_query = u'{address_data.input_string}'
                results = self.client.geocode(address_query)
                print(f'{type(self).__name__} request sent.' \
                       'Waiting on rate limit...')
                return results
            # reverse geocoding
            elif int(params["options"]) == 3:
                latitude = address_data.latitude
                longitude = address_data.longitude
                results = self.client.reverse_geocode(latitude,
                                                      longitude,
                                                      language='eng',
                                                      no_annotations='1')
                print(f'<{type(self).__name__} request sent.' \
                       'Waiting on rate limit...>')
                return results
        except RateLimitExceededError as rate_err:
            print(rate_err)
        except UnknownError as unkwn_err:
            print("\n Error: An unspecified server issue occured \n")
            print(unkwn_err)
        except InvalidInputError as invalid_err:
            print(invalid_err)
            raise

    def forward_geocode(self, params, address_input_data):
        """ 
        Reponsible for forward geocoding input addresses in stream or batch form.
        
        returns a list containing a single Address object for stream input 
        and multiple for batch input.
        """
        processed_address_list = []
        for address in address_input_data:
            result = self.send_request(params, address)
            self.num_addresses_processed += 1
            if result and len(result):
                address.latitude = result[0]['geometry']['lat']
                address.longitude = result[0]['geometry']['lng']
                address.is_valid = True
            else:
                address.is_valid = False
            processed_address_list.append(address)

        print(f'< {self.num_addresses_processed} addresses processed >')
        return processed_address_list

    def reverse_geocode(self, params, address_input_data):
        """ 
        Reponsible for forward geocoding input addresses in stream or batch form.
        
        returns a list containing a single Address object for stream input 
        and multiple for batch input.
        """
        processed_address_list = []
        for address in address_input_data:
            result = self.send_request(params, address)
            self.num_addresses_processed += 1
            if result and len(result):
                address.line_1 = result[0]['formatted']
                address.is_valid = True
            else:
                address.is_valid = False
            processed_address_list.append(address)

        print(f'< {self.num_addresses_processed} addresses processed >')
        return processed_address_list
Пример #21
0
    def execute(trial = False):
        print('finding streetlight collisions')

        def select(R, s):
            return [t for t in R if s(t)]
        def project(R, p):
            return [p(t) for t in R]
            

        print("in trafficlights collision transformation")
        
        # { bike collisions : mode_type = bike
          #   "dispatch_ts": "2015-01-01 00:24:27",
          #   "mode_type": "mv",
          #   "location_type": "Intersection",
          #   "street": "",
          #   "xstreet1": "TRAIN ST",
          #   "xstreet2": "WESTGLOW ST",
          #   "x_cord": 777243.68,
          #   "y_cord": 2930930.11,
          #   "lat": 42.2897498978,
          #   "long": -71.0525153263
          # },
        
        
        startTime = datetime.datetime.now()
        
        # Set up the database connection.
        client = dml.pymongo.MongoClient()
        repo = client.repo
        repo.authenticate('nhuang54_tkixi_wud', 'nhuang54_tkixi_wud')

        bc = repo.nhuang54_tkixi_wud.boston_collisions
        bt = repo.nhuang54_tkixi_wud.boston_streetlights


        # Boston Collisions 
        # mode_type, xstreet1, xstreet2
        bostonCollisions = bc.find()
        print("###PRINTED Bike Collisions###")

        # select to get all bike collisions
        bikeCollisions = select(bostonCollisions, lambda x: x['mode_type'] == 'bike')

        collision_filter = lambda x: {'xstreet1': x['xstreet1'],
                                     'xstreet2': x['xstreet2'],
                                     'location_type': x['location_type']
                                     }
        collision_project = project(bikeCollisions, collision_filter)
        # {'xstreet1': 'OLNEY ST', 
        #  'xstreet2': 'INWOOD ST', 
        #  'location_type': 'Intersection'}



        # # Boston Street Lights
       
        # {"the_geom" : "POINT (-71.07921632936232 42.35482231438127)", 
        #   "OBJECTID" : 10, 
        #   "TYPE" : "LIGHT", 
        #   "Lat" : 42.3548223144, 
        #   "Long" : -71.0792163294
        #   }
           
        api_key = dml.auth['services']['openCagePortal']['api_key']
        geocoder = OpenCageGeocode(api_key)
        api_limit = 0


        data = []
        streetLights = bt.find()

        for x in streetLights:
            if trial:
                if api_limit > 50:
                    break
            if api_limit > 500:
                break
            lat = x['Lat']
            lng = x['Long']
            print('lat', lat)
            api_limit+=1
            results = geocoder.reverse_geocode(lat, lng)
            print('printing results')
            if 'road' in results[0]['components']:            
                road = results[0]['components']['road']
                road = road.replace('Street', 'St')
                road = road.replace('Drive', 'Dr')
                road = road.replace('Avenue', 'Ave')
                road = road.replace('Court', 'Ct')
                road = road.replace('Highway', 'Hwy')
                road = road.replace('Parkway', 'Pkwy')
                road = road.replace('Road', 'Rd')
                road = road.replace('Boulevard', 'Blvd')
                road = road.upper()
            else:
                continue
            
            for y in collision_project:
                xstreet1 = str(y['xstreet1'])
                xstreet2 = str(y['xstreet2'])
                if road in xstreet1 or road in xstreet2:
                    streetlight_collisions = {}
                    streetlight_collisions['streetlight'] = 1
                    streetlight_collisions.update({'road': road})
                    data.append(streetlight_collisions)

        c = defaultdict(int)
        # how many accidents have happened at each intersection
        for d in data:
            c[d['road']] += d['streetlight'] 


        e = defaultdict(int)
        for f in collision_project:
            e[f['xstreet1']] += 1
            e[f['xstreet2']] += 1
        # print(e)

        data2 = []
        streetlight_collision_data = [{'road': road, 'streetlight': streetlight} for road, streetlight in c.items()]
        # print(streetlight_collision_data)
        for x in streetlight_collision_data:
            if x['road'] in e:
                # print(e)
                match = {}
                match.update(x)
                match.update({'collisions': e.get(x['road'])})
                data2.append(match)
        # print(data2)
        
        # data2 = [{'road': 'MARLBOROUGH ST', 'streetlight': 126, 'collisions': 6}, {'road': 'BACK ST', 'streetlight': 6, 'collisions': 3}, {'road': 'BEACON ST', 'streetlight': 1113, 'collisions': 43}, {'road': 'SAINT BOTOLPH ST', 'streetlight': 54, 'collisions': 6}, {'road': 'HUNTINGTON AVE', 'streetlight': 726, 'collisions': 26}, {'road': 'RING RD', 'streetlight': 4, 'collisions': 4}, {'road': 'EXETER ST', 'streetlight': 63, 'collisions': 7}, {'road': 'BOYLSTON ST', 'streetlight': 1326, 'collisions': 34}, {'road': 'NEWBURY ST', 'streetlight': 480, 'collisions': 10}, {'road': 'DARTMOUTH ST', 'streetlight': 546, 'collisions': 13}, {'road': 'COMMONWEALTH AVE', 'streetlight': 2400, 'collisions': 60}, {'road': 'SAINT JAMES AVE', 'streetlight': 64, 'collisions': 4}, {'road': 'STUART ST', 'streetlight': 49, 'collisions': 7}, {'road': 'CLARENDON ST', 'streetlight': 84, 'collisions': 3}]

        repo.dropCollection("nhuang54_tkixi_wud.streetlight_collisions")
        repo.createCollection("nhuang54_tkixi_wud.streetlight_collisions")

        repo['nhuang54_tkixi_wud.streetlight_collisions'].insert_many(data2)
        print("Done with Transformation of Traffic Lights + Bike Collisions")

        repo.logout()
        endTime = datetime.datetime.now()
        return {"start":startTime, "end":endTime}
Пример #22
0
class Scrap:
    def __init__(self):
        with open('config.json') as file:
            config = json.load(file)
        self.event_labels = config["event_labels"]
        self.geocoder = OpenCageGeocode(API_KEY)
        self.p = PyJsParser()

    def parse_date(self, date):
        try:
            day, month, year = date[0]['value'].split('.')
            hour, minute = date[2]['value'].split(':')
        except (AttributeError, ValueError) as e:
            return False
        return datetime.datetime(int(year), int(month), int(day), int(hour),
                                 int(minute), 0)

    def create_geojson(self, query=None, latlng=None):
        results = []
        if query != None:
            geo = self.geocoder.geocode(query)
            results.append(geo[0]['geometry']['lng'])
            results.append(geo[0]['geometry']['lat'])
        else:
            results.append(latlng[1])
            results.append(latlng[0])
        return Point((results[0], results[1]))

    def get_address(self, latlng):
        return self.geocoder.reverse_geocode(latlng[0],
                                             latlng[1])[0]['formatted']

    def event_parser(self, event):
        source = urllib.request.urlopen(event)
        soup = bs.BeautifulSoup(source, 'lxml')
        info = soup.find_all('script')[8].getText()
        info_dict = self.p.parse(info)

        name = info_dict['body'][4]['declarations'][0]['init']['properties'][
            2]['value']['value']
        url = info_dict['body'][4]['declarations'][0]['init']['properties'][5][
            'value']['value']
        longDescription = info_dict['body'][4]['declarations'][0]['init'][
            'properties'][6]['value']['value']
        sDF = bs.BeautifulSoup(longDescription, 'lxml')
        longDescription = str(longDescription)
        sDF = ''.join(sDF.find_all(text=True))
        shortDescription = ' '.join(re.split(r'(?<=[.:;])\s',
                                             sDF)[:2]) + ' [...] '
        tags = info_dict['body'][4]['declarations'][0]['init']['properties'][
            37]['value']['value']
        creationDate = datetime.datetime.now()
        eventStart = self.parse_date(
            info_dict['body'][4]['declarations'][0]['init']['properties'][16]
            ['value']['elements'])
        if self.parse_date(info_dict['body'][4]['declarations'][0]['init']
                           ['properties'][17]['value']['elements']):
            eventEnd = self.parse_date(
                info_dict['body'][4]['declarations'][0]['init']['properties']
                [17]['value']['elements'])
        else:
            eventEnd = eventStart
        owner = 1
        categories_elements = info_dict['body'][4]['declarations'][0]['init'][
            'properties'][18]['value']['elements']
        categories = []
        for category in categories_elements:
            # categories.append(category['value'].lower())
            if 'sport' in category['value'].lower():
                categories.append(1)
            elif 'kultura' in category['value'].lower():
                categories.append(2)
            elif 'koncert' in category['value'].lower():
                categories.append(3)
            elif 'targi' in category['value'].lower():
                categories.append(4)
            elif 'hackathon' in category['value'].lower():
                categories.append(6)
            else:
                categories.append(5)
        categories = list(set(categories))
        imageSource = info_dict['body'][4]['declarations'][0]['init'][
            'properties'][20]['value']['value']
        latlng = (float(info_dict['body'][4]['declarations'][0]['init']
                        ['properties'][31]['value']['elements'][0]['value']),
                  float(info_dict['body'][4]['declarations'][0]['init']
                        ['properties'][31]['value']['elements'][1]['value']))
        #----------- ODZNACZYĆ !!! - ograniczenie 2500 requestów/dzień
        geoJSON = str(self.create_geojson(latlng=latlng))
        address = self.get_address(latlng)

        def date_converter(o):
            if isinstance(o, datetime.datetime):
                return o.isoformat()

        used_var_list = [
            name, shortDescription, longDescription, creationDate, eventStart,
            eventEnd, owner, geoJSON, imageSource, address, self.addressCity
        ]
        dic = {}
        dic["event"] = dict(zip(self.event_labels, used_var_list))
        dic["categories"] = categories
        return json.dumps(dic, default=date_converter)

    def scrap_kiwiportal(self, url):
        self.addressCity = url.rsplit('/', 1)[-1].capitalize()
        try:
            source = urllib.request.urlopen(url)
        except:
            print('Website ERROR')
        soup = bs.BeautifulSoup(source, 'lxml')
        event_list = soup.find_all(
            'a', {
                'href':
                re.compile(
                    r'https:\/\/www\.kiwiportal\.pl\/wydarzenia/[0-9]+.*')
            })
        event_list = list(set([event['href'] for event in event_list]))
        json_list = []
        for event in event_list:
            try:
                json_list.append(self.event_parser(event))
            except AttributeError:
                print('AttributeError occured inside event_parser')
        return json_list


# test--------------------------------------------------------------------------------------------------------
# s = Scrap()
# s.scrap_kiwiportal('https://www.kiwiportal.pl/wydarzenia/m/warszawa')
# s.create_geojson(query='Polska, Poznań, ulica Stróżyńskiego 17c/10')
# print(s.get_address((21.0246, 52.2791)))
Пример #23
0
          (16.661595773851438, 51.23220045671269),
          (25.848726239888855, 51.2126706577948),
          (62.216195513463404, 113.05628879578387),
          (-16.910570007011934, 30.596807715281102),
          (-17.70870157276752, 168.357288837245),
          (19.04098614899285, 55.481410272456614),
          (41.882017240235655, 45.93406092647046),
          (9.948862943368631, -83.49190467887524),
          (14.020642002077539, -86.6581308772755),
          (0.4073893927731693, 41.335555895737784),
          (-17.32506818593016, -63.57161999641103),
          (39.663907642111226, 44.89402557511953),
          (18.34190344872168, 49.88539663682489),
          (49.26081518856387, 39.55267665158491),
          (43.72471712631733, 19.312826677730367),
          (39.482563391305106, 29.289350673373175),
          (-1.5830518037832224, 30.412157237720475),
          (-0.9218881032413536, 31.682955507638592)]
key = "[REDACTED]"
geocoder = OpenCageGeocode(key)

countries = list()

for i in coords:
    result = geocoder.reverse_geocode(i[0], i[1])
    ans = result[0]['components']['country']
    countries.append(ans[0])
    print(ans)

print(countries)
Пример #24
0
def emergency(request):

    if request.method == "POST":
        message = request.POST['message']
        ip = request.POST['ip']

        key = "0f9f6b4b15d7c900a25be24e309d5b99"
        # print("hi")
        # print(x.ip)
        url = "http://api.ipstack.com/" + ip + "?access_key=" + key

        response = requests.get(url).json()
        # print(response)
        lat_user = response['latitude']
        lon_user = response['longitude']
        print(lat_user)
        print(lon_user)
        from opencage.geocoder import OpenCageGeocode
        key = '8c0d6897d5334facb9c5419400ff2dab'

        geocoder = OpenCageGeocode(key)

        results = geocoder.reverse_geocode(lat_user, lon_user)
        print(results[0]['formatted'])
        add = results[0]['formatted']
        message = message + "Address- " + add
        radius = 10000
        car = 1
        user_lat = radians(response['latitude'])
        user_lon = radians(response['longitude'])

        data = Profile.objects.all()

        R = 6373.0

        ls = []
        print(radius)
        for x in data:
            lat_user = radians(19.1778797)
            ip = x.ip
            key = "0f9f6b4b15d7c900a25be24e309d5b99"
            print("hi")
            # print(x.ip)
            url = "http://api.ipstack.com/" + ip + "?access_key=" + key

            response = requests.get(url).json()
            # print(response)
            lat = response['latitude']
            lon = response['longitude']

            lat_b = radians(lat)
            long_diff = radians(72.8733183 - lon)
            distance = (sin(lat_user) * sin(lat_b) +
                        cos(lat_user) * cos(lat_b) * cos(long_diff))
            resToMile = degrees(acos(distance)) * 69.09
            resToMt = resToMile / 0.00062137119223733

            # print(resToMt)

            if resToMt < radius:
                temp = {}
                temp['name'] = x.name
                temp['contact'] = x.contact
                temp['address'] = x.address

                ip = x.ip

                temp['latitude'] = lat
                temp['longitude'] = lon

                temp['id'] = x.id

                temp['bloodgroup'] = x.bloodgroup

                ls.append(temp)

        print(ls)

        print(car)

        contact_list = []

        for x in ls:
            temp = "+" + str(x['contact'])
            contact_list.append(temp)
            print(x['contact'])

        message_to_broadcast = (message)
        client = Client("AC9d8a94f846d748f362e6dda727e83eaa",
                        "ec46f7a3a849a73af6db6874516a2c3b")
        for recipient in contact_list:
            if recipient:
                client.messages.create(to=recipient,
                                       from_="+14422426473",
                                       body=message_to_broadcast)

        return render(request, 'emergencymap.html', {
            "data": ls,
            'rad': radius,
            'temp': car
        })

    return render(request, 'emergencyform.html')
Пример #25
0
def get_county(lat, lng):
    # Allow running with multiple pairs to run them in parallel (at least to try)
    cgc = OpenCageGeocode(key)
    resp = cgc.reverse_geocode(lat, lng, no_annotations=1)
    return resp[0]['components']['county']
Пример #26
0
 def get_city_name_from_coordinates(self, lat: float, long: float) -> str:
     geocoder = OpenCageGeocode(self.key)
     res = geocoder.reverse_geocode(lat, long)
     return res[0]['components']['city']
Пример #27
0
class Geolocate:
    def __init__(self, geocodeToken, blackThreshold=1):
        # create geocode instance
        self.geocoder = OpenCageGeocode(str(geocodeToken))

        # load map as np array
        equirectangularMap = Image.open('equirectangularMap.png').convert('L')
        self.npMap = np.array(equirectangularMap)

        # convert grayscale value into binary state (land or water)
        for i in range(0, self.npMap.shape[0]):
            for j in range(0, self.npMap.shape[1]):
                if self.npMap[i, j] >= blackThreshold:
                    self.npMap[i, j] = 1
                else:
                    self.npMap[i, j] = 0

    def getCoordFromLoc(self, location):
        try:
            locData = self.geocoder.geocode(location)
            return (locData[0]['geometry']['lng'],
                    locData[0]['geometry']['lat'])
        except:
            return 'error'

    def getLocFromCoord(self, x, y):
        try:
            locData = self.geocoder.reverse_geocode(y, x)
            print(locData[0].get('formatted'))
            return locData[0].get('formatted')
        except:
            return 'error'

    def getMap(self, viewGraph=False, points=[(0, 0)]):
        if viewGraph:
            plt.imshow(self.npMap, cmap=plt.cm.binary)
            for p in points:
                i, j = xyToij(*p)
                plt.scatter(j, i)
            plt.show()
        return self.npMap

    def isLand(self, x, y):
        #find array position from location and check if k=1 (isLand?)
        i, j = xyToij(x, y)
        if self.npMap[i, j] == 1:
            return True
        else:
            return False

    def findClosestOcean(self, x, y):
        if self.isLand(x, y):
            minDistanceIndexPair = [x**2 + y**2, (0, 0)]
            for i in range(0, self.npMap.shape[0]):
                for j in range(0, self.npMap.shape[1]):
                    #check if new coordinate is ocean and is closer to the target than the previous closest
                    if self.npMap[i, j] == 0 and euclideanDistanceSquared(
                        (x, y), ijToxy(i, j)) < minDistanceIndexPair[0]:
                        minDistanceIndexPair[0] = euclideanDistanceSquared(
                            (x, y), ijToxy(i, j))
                        minDistanceIndexPair[1] = (i, j)
            return (ijToxy(*minDistanceIndexPair[1]))
        else:
            return (x, y)
Пример #28
0
def adddetail(request, name, lat, lng):

# TRIGGER SHOWING INSTRUCTIONS
    user_activity = MasterAddModel.objects.filter(user=request.user).count()
    print("user activity = " + str(user_activity))

    try:
        notification_trigger = Notification.objects.get(name="Adding Spots").trigger_max
    except:
        notification_trigger = None

    try:
        notification = Notification.objects.get(name="Adding Spots")
    except:
        notification=False


####FOURSQUARE API####
    lat = lat
    long = lng
    targetLocation = str(lat) + ', ' + str(long)

    detailsURL = 'https://api.foursquare.com/v2/venues/suggestcompletion'
    detailsParams = dict(
        client_id='0PR1PTLMSLBM0ORYW5U2YGL43IOZXFVKWFIC2DHHXOP30Z35',
        client_secret='SJDG5K1D5NARSRZYAAYPJMTJBPIGW4ONUTQBT4HTDNUGSLQQ',
         v='20180323',
         ll=targetLocation,
         query=name,
         limit=10,
         )
    detailsResponse = requests.get(url=detailsURL, params=detailsParams)
    detailsData = detailsResponse.json()
    detailsStatus = detailsResponse.status_code
    detailsInfo = json.loads(detailsResponse.text)

    detailsResult = detailsInfo['response']['minivenues'][0]
    resultName = detailsResult['name']
    try:
        resultAddress = detailsResult['location']['address']
    except:
        resultAddress = ''
    try:
        resultCity = detailsResult['location']['city']
    except:
        resultCity = ''
    try:
        resultCountry = detailsResult['location']['country']
    except:
        resultCountry = ''
    try:
        resultCategory1 = detailsResult['categories'][0]['name']
    except:
        resultCategory1 = ""
    try:
        resultCategory2 = detailsResult['categories'][1]['name']
    except:
        resultCategory2 = ""
    try:
        resultCategory3 = detailsResult['categories'][2]['name']
    except:
        resultCategory3 = ""
    try:
        resultPostcode = detailsResult['location']['postalCode']
    except:
        resultPostcode = ""

    imageQuery = resultName + " " + resultAddress

####GOOGLE IMAGE API ######
    searchURL = 'https://www.googleapis.com/customsearch/v1'
    searchParams = dict(
        cx=google_project_cx,
        key=google_dev_api_key,
        q=imageQuery,
        searchType='image',
        fileType='.jpg',
        num=6,
    )

    searchResponse = requests.get(url=searchURL, params=searchParams)
    searchStatus = searchResponse.status_code
    searchData = searchResponse.json()
    searchInfo = json.loads(searchResponse.text)

    try:
        imageResult = searchInfo['items']
    except:
        imageResult = None

###ADD ADDITIONAL LOCATION DETAILS####
    key = 'a98d10680c0c41d082d9de1c23dcec22'
    geocoder = OpenCageGeocode(key)

    locationResponse = geocoder.reverse_geocode(lat, long)

    try:
        resultSuburb = locationResponse[0]['components']['suburb']
    except:
        resultSuburb = ''

####FORM FOR USERS INPUTS ####
    if request.method == 'POST':
        form = MasterAddForm(request.POST, request.FILES)
        if form.is_valid():
            post = form.save(commit=False)
            post.user = request.user
            post.name = resultName
            post.city = resultCity
            post.address = resultAddress
            post.category1 = resultCategory1
            post.category2 = resultCategory2
            post.category3 = resultCategory3
            post.postcode = resultPostcode
            post.suburb = resultSuburb
            post.lat = lat
            post.long = long
            post.save()
            return redirect('browsespots') #or whatever the url
    else:
        form = MasterAddForm()

    friends_here = CleanReviewModel.objects.filter(name=name).exclude(user=request.user).count()

    context = {
    'notification': notification,
    'notification_trigger': notification_trigger,
    'user_activity': notification_trigger,
    'form': form,
    'name': name,
    'targetLocation': targetLocation,
    'detailsData': detailsData,
    'resultName': resultName,
    'resultCity': resultCity,
    'resultCountry': resultCountry,
    'resultAddress': resultAddress,
    'resultCategory1': resultCategory1,
    'resultCategory2': resultCategory2,
    'resultCategory3': resultCategory3,
    'resultPostcode': resultPostcode,
    'searchStatus': searchStatus,
    'imageResult': imageResult,
    'searchData': searchData,
    'resultSuburb': resultSuburb,
    'searchStatus': searchStatus,
    'searchData': searchData,
    'friends_here': friends_here,
    'resultName': resultName,
    }

    return render(request, 'adddetail.html', context=context)
Пример #29
0
    def execute(trial=False):
        def select(R, s):
            return [t for t in R if s(t)]

        def project(R, p):
            return [p(t) for t in R]

        # gets rid of values that don't make sense: installed at year 0
        def ridOf0(t):
            return t['installed'] != 0

        print("in transformation 1")

        startTime = datetime.datetime.now()

        # Set up the database connection.
        client = dml.pymongo.MongoClient()
        repo = client.repo
        repo.authenticate('tkixi', 'tkixi')

        bl = repo.tkixi.boston_bikes
        bh = repo.tkixi.boston_hubway

        # Boston Bike Network
        # InstalledAt, Street Name
        bikeNetwork = bl.find()
        print("###PRINTED BIKE NETWORK###")

        # project for street name and date of installment
        bike_filter = lambda x: {
            'street': x['STREET_NAM'],
            'installed': x['InstallDat']
        }
        bike_project = project(bikeNetwork, bike_filter)
        # remove duplicate streets
        no_dupes = [
            i for n, i in enumerate(bike_project)
            if i not in bike_project[n + 1:]
        ]

        # select to get rid of entries with installed at year 0
        clean_data = select(no_dupes, ridOf0)
        bike_data = clean_data
        # data is list

        # Boston Hubway Station
        # Convert lat lng to street name
        hubwayStation = bh.find()
        print("###PRINTED HUBWAY STATIONS###")
        api_key = dml.auth['services']['openCagePortal']['api_key']
        geocoder = OpenCageGeocode(api_key)

        # dictionary of street names with hubway stations in Boston
        selected = {}
        api_limit = 0
        for x in hubwayStation:
            lat = x['Latitude']
            lng = x['Longitude']
            api_limit += 1
            results = geocoder.reverse_geocode(lat, lng)

            if 'road' in results[0]['components']:
                street = results[0]['components']['road']
                selected.update({street: 1})
            if api_limit >= 2500:  # exceed api requests per day
                break
            # print(results)
            # print(selected)

        print("Length of selected", len(selected))
        print("unique streets", len(bike_data))

        # compare dictionary of hubway station street names with bike network data
        hubwayNetwork = []
        # hasHubwayStation = 1 is yes and 0 is no
        # print('bike data', len(bike_data))
        for x in bike_data:
            # print(x)
            if x['street'] in selected:
                # print('x',x)
                x.update({'hasHubwayStation': 1})
                hubwayNetwork.append(x)
            else:
                x.update({'hasHubwayStation': 0})
                hubwayNetwork.append(x)
        # print(hubwayNetwork)

        repo.dropCollection("tkixi.hubway_network")
        repo.createCollection("tkixi.hubway_network")

        repo['tkixi.hubway_network'].insert_many(hubwayNetwork)
        print("Done with Transformation 1")

        repo.logout()
        endTime = datetime.datetime.now()
        return {"start": startTime, "end": endTime}
Пример #30
0
    def execute(trial=False):
        def select(R, s):
            return [t for t in R if s(t)]

        def project(R, p):
            return [p(t) for t in R]

        print("in transformation 3")

        startTime = datetime.datetime.now()

        # Set up the database connection.
        client = dml.pymongo.MongoClient()
        repo = client.repo
        repo.authenticate('tkixi', 'tkixi')

        bc = repo.tkixi.boston_collisions
        bw = repo.tkixi.boston_weather

        # Boston Collisions
        # mode_type, xstreet1, xstreet2
        bostonCollisions = bc.find()
        print("###PRINTED Bike Collisions###")

        # select to get all bike collisions
        bikeCollisions = select(bostonCollisions,
                                lambda x: x['mode_type'] == 'bike')

        # get lat lng
        collision_filter = lambda x: {
            'timestamp': x['dispatch_ts'],
            'lat': x['lat'],
            'lng': x['long']
        }
        collision_project = project(bikeCollisions, collision_filter)
        # {"dispatch_ts": "2015-01-01 03:50:33",
        # "lat": 42.2570810965,
        # "long": -71.1201108871
        #  }
        api_key = dml.auth['services']['openCagePortal']['api_key']
        geocoder = OpenCageGeocode(api_key)

        # dictionary of street names with hubway stations in Boston
        selected = {}
        api_limit = 0
        collisionCity = []
        for x in collision_project:
            lat = x['lat']
            lng = x['lng']
            api_limit += 1
            # x.update({'city': 'Boston'})
            # collisionCity.append(x)

            # reverse geocode the lat lng of the collision and cross reference the city with city's weather
            results = geocoder.reverse_geocode(lat, lng)
            if 'city' in results[0]['components']:
                x.update({'city': (results[0]['components']['city'])})
                collisionCity.append(x)

            if api_limit >= 2500:  # exceed api requests per day -- reduce to 100 if you want it to run faster
                break

        # Boston Weather
        #  {
        #   "STATION": "USW00014739",
        #   "NAME": "BOSTON, MA US",
        #   "DATE": "2015-01-01",
        #   "AWND": 14.32,
        #   "PGTM": "",
        #   "PRCP": 0,
        #   "SNOW": 0,
        #   "TAVG": 26,
        #   "TMAX": 33,
        #   "TMIN": 22,
        #   "WDF2": 230,
        #   "WDF5": 210,
        #   "WSF2": 23,
        #   "WSF5": 29.1,
        #   "WT01": "",
        #   "WT02": "",
        #   "WT03": "",
        #   "WT04": "",
        #   "WT05": "",
        #   "WT06": "",
        #   "WT08": "",
        #   "WT09": ""
        # },
        bostonWeather = bw.find()

        collision_weather = []
        # iterate boston's weather
        for k in bostonWeather:
            for l in collisionCity:
                date = l['timestamp'].split()[0]
                city = k['NAME'].split()[0][:-1]
                if l['city'].upper() == city and date == k['DATE']:
                    # found collision day's weather
                    # we want to find out average wind speed and the month
                    month = date[5:7]  #month
                    wind_collisions = {}
                    wind_collisions.update({'month': month, 'wind': k['AWND']})
                    collision_weather.append(wind_collisions)
        # pprint(collision_weather)
        months = [
            '01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11',
            '12'
        ]
        avg_wind = {}
        monthly_wind = []
        for x in months:
            count = 0
            total_wind = 0
            avg_wind = {}
            for y in collision_weather:
                if y['month'] == x:
                    count += 1
                    total_wind += y['wind']
                    avg_wind.update({
                        'month': x,
                        'wind': total_wind,
                        'accidents': count
                    })
                    monthly_wind.append(avg_wind)

        # averaging wind speed
        minify = []

        for x in months:
            for y in monthly_wind:
                if x == y['month']:
                    avg = y['wind'] / y['accidents']
                    minify.append({
                        'month': x,
                        'average_wind': avg,
                        'accidents': y['accidents']
                    })
        result = [
            dict(tupleized) for tupleized in set(
                tuple(item.items()) for item in minify)
        ]
        pprint(result)

        repo.dropCollection("tkixi.weather_collision")
        repo.createCollection("tkixi.weather_collision")

        repo['tkixi.weather_collision'].insert_many(result)
        print("Done with Transformation 3")

        repo.logout()
        endTime = datetime.datetime.now()
        return {"start": startTime, "end": endTime}
Пример #31
0
class getBiodiversity():
    def __init__(self, url, key, taxonomy_columns, location_columns):
        self.url = url
        self.geocoder = OpenCageGeocode(key)
        self.TAXONOMY_COLUMNS = taxonomy_columns
        self.LOCATION_COORDINATES = location_columns
        try:
            self.df_data = pd.read_csv(url,
                                       sep=';',
                                       header=0,
                                       encoding='utf-8')
        except Exception as e:
            self.df_data = pd.DataFrame()
            print("Aborting... couldn't read this file: %s" % url)
            print(e.args)
        return None

    def getColumns(self):
        self.df_columns = list(self.df_data.columns)
        return None

    def checkEmpty(self):
        self.getColumns()
        self.df_dataNAN = pd.DataFrame(
            np.where(
                (self.df_data == '') | (self.df_data == 'Sem Informações'), 1,
                0))
        self.df_dataNAN.columns = self.df_columns
        self.df_dataNAN = 100 * self.df_dataNAN.mean()
        return None

    def getLastFilled(self, columns):
        filled_columns = [
            column for column in columns if (column != "Sem Informações")
        ]
        return 'NA' if len(filled_columns) == 0 else self.TAXONOMY_COLUMNS[
            len(filled_columns) - 1]

    def addTaxonomicLevel(self, col_name):
        self.df_data[col_name] = self.df_data[self.TAXONOMY_COLUMNS].apply(
            lambda x: self.getLastFilled(x), axis=1)
        self.df_taxonomy_info = self.df_data[col_name].value_counts()
        return None

    def extractTaxonomy(self, columns):
        self.df_taxonomy = self.df_data[columns].copy()
        return None

    def getTaxonomy(self, col_name='taxonomic_level'):
        self.addTaxonomicLevel(col_name)
        self.extractTaxonomy(self.TAXONOMY_COLUMNS + [col_name])
        return None

    def filterFields(self, columns, values):
        #filter = np.logical_and.reduce([self.df_data[columns[i]].isin(values[i]) for i in range(len(columns))])
        filter = np.sum([
            self.df_data[columns[i]].isin(values[i]) + (len(values[i]) == 0)
            for i in range(len(columns))
        ],
                        axis=0) == len(columns)
        if columns:
            self.df_filtered = self.df_data[filter].copy()
        else:
            self.df_filtered = self.df_data.copy()
        return None

    def parseFloat(self, info):
        value = float(info)
        try:
            value = float(info)
        except:
            value = 0.0
        return value

    def checkGeoInfo(self, components, reported):
        aux = []
        unmatched = 0
        for elem in ["country", "state", "state_code", "city"]:
            try:
                value = components[elem]
            except:
                value = "NA"
            aux.append(value)
        unmatched += 1 if reported[0] != aux[0] else 0
        unmatched += 1 if not reported[1] in [aux[1], aux[2]] else 0
        unmatched += 1 if reported[2] != aux[3] else 0
        return unmatched

    def reverseGeocode(self, latlon):
        geo = self.geocoder.reverse_geocode(latlon[0],
                                            latlon[1],
                                            no_annotations='1',
                                            pretty='1',
                                            language='pt')
        comp = geo[0]['components']
        info = self.checkGeoInfo(comp, [latlon[2], latlon[3], latlon[4]])
        return pd.Series((geo[0]['formatted'], info))

    def setMapZoom(self, coords):
        try:
            rangelat = math.sqrt(170 / (max(coords[0][:]) - min(coords[0][:])))
            rangelon = math.sqrt(360 / (max(coords[1][:]) - min(coords[1][:])))
            zoom = int(min(rangelat, rangelon)) + 1
        except:
            zoom = 1
        return zoom

    def printMap(self):
        coords = self.df_location_sample[[
            "AdjustedLatitude", "AdjustedLongitude", "ReversedAddress",
            "Confidence"
        ]].T.values.tolist()
        COLORS = ['green', 'lightgreen', 'orange', 'red']
        center = [mean(coords[0][:]), mean(coords[1][:])]
        zoom = self.setMapZoom(coords[0:2][:])
        my_map = folium.Map(location=center,
                            zoom_start=zoom,
                            tiles="OpenStreetMap")
        for i in range(len(self.df_location_sample)):
            folium.Marker(location=[coords[0][i], coords[1][i]],
                          popup=coords[2][i],
                          icon=folium.Icon(color=COLORS[coords[3][i]],
                                           icon='map-marker')).add_to(my_map)
        self.observations_map = my_map
        return None

    def checkCoordinates(self, size):
        self.df_filtered["AdjustedLatitude"] = self.df_data["Latitude"].apply(
            lambda x: self.parseFloat(x))
        self.df_filtered["AdjustedLongitude"] = self.df_data[
            "Longitude"].apply(lambda x: self.parseFloat(x))
        if len(self.df_filtered) < size:
            print("Not enough data to show. Please check your filter opetions")
            self.df_location_sample = pd.DataFrame()
            self.observations_map = None
            return None
        self.df_location_sample = self.df_filtered.sample(n=size).copy()
        self.df_location_sample[[
            'ReversedAddress', 'Confidence'
        ]] = self.df_location_sample[['AdjustedLatitude', 'AdjustedLongitude']
                                     + self.LOCATION_COORDINATES].apply(
                                         self.reverseGeocode, axis=1)
        self.printMap()
        return None