Esempio n. 1
0
    def start(self):
        total_count = 0
        error_count = 0
        success_count = 0
        failed_count = 0

        rows = get_messages_for_geocoding(self.pg_conn, self.limit)

        #print "[geocode.Geocode]: - INFO - found %s rows to geocode" % len(rows)

        for row in rows:
            total_count += 1

            location = row['geo_location']
            message_id = row['id']
            time = row['time']
            url_id = row['url_id']

            try:
                result = geocodeLocation(location, self.mongo_conn.geocode)
                #print "[geocode.Geocode]: (%d invalid; %d total; (%.1f%%)) %s\t\t%s\n" % (error_count, total_count, (float(error_count) / float(total_count) * 100), location, str(result))

                if result:
                    lat = result['lat']
                    lon = result['lon']

                    if lat is not None and lon is not None:
                        save_message_location(self.pg_conn, message_id, lat,
                                              lon)

                        county = None
                        city = None
                        neighborhood = None

                        county, city, neighborhood = save_geography_for_message_id(
                            self.pg_conn, message_id)

                        if county or city or neighborhood:
                            # add to timeseries
                            hour = 60 * 60
                            hours = int(time / hour)
                            increment_geo_timeseries(self.pg_conn,
                                                     url_id,
                                                     hours,
                                                     county=county,
                                                     city=city,
                                                     neighborhood=neighborhood)

                    success_count += 1

                else:
                    print '[geocode.Geocode]: - INFO - error: %s' % location
                    error_count += 1
                    save_message_invalidlocation(self.pg_conn, message_id)
            except:
                print '[geocode.Geocode]: - WARNING - failed: %s' % location
                failed_count += 1

        print "\n[geocode.Geocode]: - INFO - finished: Total: %s, Successful: %s, Errors: %s, Failed: %s" % (
            total_count, success_count, error_count, failed_count)
def run_geocoding(pgConn, mongoCollection, url_id, message_id, tweet_time,
                  geo_location):
    try:
        result = geocodeLocation(geo_location, mongoCollection)

        if result:
            lat = result['lat']
            lon = result['lon']

            if lat is not None and lon is not None:
                print "[Geocoding]: %s - Info - Saving lat / lng for %s" % (
                    getCurrentTime(), geo_location)
                save_message_location(pgConn, message_id, lat, lon)

                county, city, neighborhood = save_geography_for_message_id(
                    pgConn, message_id)

                if county or city or neighborhood:
                    update_time_series(pgConn, url_id, tweet_time, county,
                                       city, neighborhood)

        else:
            print "[Geocoding]: %s - Warning - could not find lat / lng for %s " % (
                getCurrentTime(), geo_location)
            save_message_invalidlocation(pgConn, message_id)
    except:
        print "[Geocoding]: %s - Error - looking for %s" % (getCurrentTime(),
                                                            geo_location)
    finally:
        return True
Esempio n. 3
0
    def start(self):
        total_count = 0
        error_count = 0
        success_count = 0
        failed_count = 0

        rows = get_messages_for_geocoding(self.pg_conn, self.limit)

        #print "[geocode.Geocode]: - INFO - found %s rows to geocode" % len(rows)

        for row in rows:
            total_count += 1

            location    = row['geo_location']
            message_id  = row['id']
            time        = row['time']
            url_id      = row['url_id']

            try:
                result = geocodeLocation(location, self.mongo_conn.geocode)
                #print "[geocode.Geocode]: (%d invalid; %d total; (%.1f%%)) %s\t\t%s\n" % (error_count, total_count, (float(error_count) / float(total_count) * 100), location, str(result))

                if result:
                    lat = result['lat']
                    lon = result['lon']

                    if lat is not None and lon is not None:
                        save_message_location(self.pg_conn, message_id, lat, lon)

                        county = None
                        city = None
                        neighborhood = None

                        county, city, neighborhood = save_geography_for_message_id(self.pg_conn, message_id)

                        if county or city or neighborhood:
                            # add to timeseries
                            hour = 60*60
                            hours = int(time / hour)
                            increment_geo_timeseries(self.pg_conn, url_id, hours, county=county, city=city, neighborhood=neighborhood)

                    success_count += 1

                else:
                    print '[geocode.Geocode]: - INFO - error: %s' % location
                    error_count += 1
                    save_message_invalidlocation(self.pg_conn, message_id)
            except:
                print '[geocode.Geocode]: - WARNING - failed: %s' % location
                failed_count += 1

        print "\n[geocode.Geocode]: - INFO - finished: Total: %s, Successful: %s, Errors: %s, Failed: %s" % (total_count, success_count, error_count, failed_count)
def run_geocoding(pgConn, mongoCollection, url_id, message_id, tweet_time, geo_location):
    try:
        result = geocodeLocation(geo_location, mongoCollection)

        if result:
            lat = result['lat']
            lon = result['lon']

            if lat is not None and lon is not None:
                print "[Geocoding]: %s - Info - Saving lat / lng for %s" % (getCurrentTime(), geo_location)
                save_message_location(pgConn, message_id, lat, lon)

                county, city, neighborhood = save_geography_for_message_id(pgConn, message_id)

                if county or city or neighborhood:
                    update_time_series(pgConn, url_id, tweet_time, county, city, neighborhood)

        else:
            print "[Geocoding]: %s - Warning - could not find lat / lng for %s " % (getCurrentTime(), geo_location)
            save_message_invalidlocation(pgConn, message_id)
    except:
        print "[Geocoding]: %s - Error - looking for %s" % (getCurrentTime(), geo_location)
    finally:
        return True