def get_lat_long(dbo, address, town, county, postcode, country = None): """ Looks up a latitude and longitude from an address using GEOCODE_URL and returns them as lat,long,(first 3 chars of address) Returns None if no results were found. NB: dbo is only used for contextual reference in logging, no database calls are made by any of this code. """ if address.strip() == "": return None try: # Synchronise this process to a single thread to prevent # abusing our geo provider and concurrent requests for the # same address when opening an animal with the same # original/brought in by owner, etc. lat_long_lock.acquire() url = "" if country is None: country = LOCALE_COUNTRY_NAME_MAP[dbo.locale] if BULK_GEO_PROVIDER == "cloudmade": q = normalise_cloudmade(address, town, county, postcode, country) url = CLOUDMADE_URL.replace("{key}", BULK_GEO_PROVIDER_KEY).replace("{q}", q) elif BULK_GEO_PROVIDER == "nominatim": q = normalise_nominatim(address, town, county, postcode, country) url = NOMINATIM_URL.replace("{q}", q) else: al.error("unrecognised geo provider: %s" % BULK_GEO_PROVIDER, "geo.get_lat_long", dbo) al.debug("looking up geocode for address: %s" % q, "geo.get_lat_long", dbo) key = "nom:" + q if cache.available(): v = cache.get(key) if v is not None: al.debug("cache hit for address: %s = %s" % (q, v), "geo.get_lat_long", dbo) return v jr = urllib2.urlopen(url, timeout = GEO_LOOKUP_TIMEOUT).read() j = json.loads(jr) latlon = None if BULK_GEO_PROVIDER == "cloudmade": latlon = parse_cloudmade(dbo, jr, j, q) elif BULK_GEO_PROVIDER == "nominatim": latlon = parse_nominatim(dbo, jr, j, q) # Cache this address/geocode response for an hour if cache.available() and latlon is not None: cache.put(key, latlon, 3600) return latlon except Exception,err: al.error(str(err), "geo.get_lat_long", dbo) return None
def get_id(dbo, table): """ Returns the next ID in sequence for a table. Will use memcache for pk generation if the CACHE_PRIMARY_KEYS option is on and will set the cache first value if not set. If the database has an ASM2 primary key table, it will be updated with the next pk value. """ strategy = "" nextid = 0 if ASM3_PK_STRATEGY == "max" or dbo.has_asm2_pk_table: nextid = _get_id_max(dbo, table) strategy = "max" elif ASM3_PK_STRATEGY == "memcache" and cache.available(): nextid = _get_id_memcache(dbo, table) strategy = "memcache" elif ASM3_PK_STRATEGY == "pseq" and dbo.dbtype == "POSTGRESQL": nextid = _get_id_postgres_seq(dbo, table) strategy = "pseq" else: raise Exception("No valid PK strategy found") if dbo.has_asm2_pk_table: _get_id_set_asm2_primarykey(dbo, table, nextid + 1) strategy += " asm2pk" al.debug("get_id: %s -> %d (%s)" % (table, nextid, strategy), "db.get_id", dbo) return nextid
def query_cache(dbo, sql, age = 60): """ Runs the query given and caches the result for age seconds. If there's already a valid cached entry for the query, returns the cached result instead. If CACHE_COMMON_QUERIES is set to false, just runs the query without doing any caching and is equivalent to db.query() """ if not CACHE_COMMON_QUERIES or not cache.available(): return query(dbo, sql) cache_key = "%s:%s:%s" % (dbo.alias, dbo.database, sql.replace(" ", "_")) m = hashlib.md5() m.update(cache_key) cache_key = "q:%s" % m.hexdigest() results = cache.get(cache_key) if results is not None: return results results = query(dbo, sql) cache.put(cache_key, results, age) return results
def query_cache(dbo, sql, age=60): """ Runs the query given and caches the result for age seconds. If there's already a valid cached entry for the query, returns the cached result instead. If CACHE_COMMON_QUERIES is set to false, just runs the query without doing any caching and is equivalent to db.query() """ if not CACHE_COMMON_QUERIES or not cache.available(): return query(dbo, sql) cache_key = "%s:%s:%s" % (dbo.alias, dbo.database, sql.replace(" ", "_")) m = hashlib.md5() m.update(cache_key) cache_key = "q:%s" % m.hexdigest() results = cache.get(cache_key) if results is not None: return results results = query(dbo, sql) cache.put(cache_key, results, age) return results
def get_lat_long(dbo, address, town, county, postcode, country=None): """ Looks up a latitude and longitude from an address using GEOCODE_URL and returns them as lat,long,(first 3 chars of address) Returns None if no results were found. NB: dbo is only used for contextual reference in logging, no database calls are made by any of this code. """ if address.strip() == "": return None try: # Synchronise this process to a single thread to prevent # abusing our geo provider and concurrent requests for the # same address when opening an animal with the same # original/brought in by owner, etc. lat_long_lock.acquire() url = "" if country is None: country = LOCALE_COUNTRY_NAME_MAP[dbo.locale] if BULK_GEO_PROVIDER == "cloudmade": q = normalise_cloudmade(address, town, county, postcode, country) url = BULK_GEO_CLOUDMADE_URL.replace( "{key}", BULK_GEO_PROVIDER_KEY).replace("{q}", q) elif BULK_GEO_PROVIDER == "nominatim": q = normalise_nominatim(address, town, county, postcode, country) url = BULK_GEO_NOMINATIM_URL.replace("{q}", q) elif BULK_GEO_PROVIDER == "smcom": q = normalise_nominatim(address, town, county, postcode, country) url = BULK_GEO_SMCOM_URL.replace("{q}", q) elif BULK_GEO_PROVIDER == "google": q = normalise_google(address, town, county, postcode, country) url = BULK_GEO_GOOGLE_URL.replace("{q}", q) else: al.error("unrecognised geo provider: %s" % BULK_GEO_PROVIDER, "geo.get_lat_long", dbo) al.debug("looking up geocode for address: %s" % q, "geo.get_lat_long", dbo) key = "nom:" + q if cache.available(): v = cache.get(key) if v is not None: al.debug("cache hit for address: %s = %s" % (q, v), "geo.get_lat_long", dbo) return v jr = urllib2.urlopen(url, timeout=BULK_GEO_LOOKUP_TIMEOUT).read() j = json.loads(jr) latlon = None if BULK_GEO_PROVIDER == "cloudmade": latlon = parse_cloudmade(dbo, jr, j, q) elif BULK_GEO_PROVIDER == "nominatim": latlon = parse_nominatim(dbo, jr, j, q) elif BULK_GEO_PROVIDER == "smcom": latlon = parse_nominatim(dbo, jr, j, q) elif BULK_GEO_PROVIDER == "google": latlon = parse_google(dbo, jr, j, q) if BULK_GEO_SLEEP_AFTER > 0: time.sleep(BULK_GEO_SLEEP_AFTER) return latlon except Exception, err: al.error(str(err), "geo.get_lat_long", dbo) return None