def get_lat_long(dbo, address, town, county, postcode, country = None):
"""
Looks up a latitude and longitude from an address using GEOCODE_URL
and returns them as lat,long,(first 3 chars of address)
Returns None if no results were found.
NB: dbo is only used for contextual reference in logging, no database
calls are made by any of this code.
"""
if address.strip() == "":
return None
try:
# Synchronise this process to a single thread to prevent
# abusing our geo provider and concurrent requests for the
# same address when opening an animal with the same
# original/brought in by owner, etc.
lat_long_lock.acquire()
url = ""
if country is None:
country = LOCALE_COUNTRY_NAME_MAP[dbo.locale]
if BULK_GEO_PROVIDER == "cloudmade":
q = normalise_cloudmade(address, town, county, postcode, country)
url = CLOUDMADE_URL.replace("{key}", BULK_GEO_PROVIDER_KEY).replace("{q}", q)
elif BULK_GEO_PROVIDER == "nominatim":
q = normalise_nominatim(address, town, county, postcode, country)
url = NOMINATIM_URL.replace("{q}", q)
else:
al.error("unrecognised geo provider: %s" % BULK_GEO_PROVIDER, "geo.get_lat_long", dbo)
al.debug("looking up geocode for address: %s" % q, "geo.get_lat_long", dbo)
key = "nom:" + q
if cache.available():
v = cache.get(key)
if v is not None:
al.debug("cache hit for address: %s = %s" % (q, v), "geo.get_lat_long", dbo)
return v
jr = urllib2.urlopen(url, timeout = GEO_LOOKUP_TIMEOUT).read()
j = json.loads(jr)
latlon = None
if BULK_GEO_PROVIDER == "cloudmade":
latlon = parse_cloudmade(dbo, jr, j, q)
elif BULK_GEO_PROVIDER == "nominatim":
latlon = parse_nominatim(dbo, jr, j, q)
# Cache this address/geocode response for an hour
if cache.available() and latlon is not None:
cache.put(key, latlon, 3600)
return latlon
except Exception,err:
al.error(str(err), "geo.get_lat_long", dbo)
return None
def get_id(dbo, table):
"""
Returns the next ID in sequence for a table.
Will use memcache for pk generation if the CACHE_PRIMARY_KEYS option is on
and will set the cache first value if not set.
If the database has an ASM2 primary key table, it will be updated
with the next pk value.
"""
strategy = ""
nextid = 0
if ASM3_PK_STRATEGY == "max" or dbo.has_asm2_pk_table:
nextid = _get_id_max(dbo, table)
strategy = "max"
elif ASM3_PK_STRATEGY == "memcache" and cache.available():
nextid = _get_id_memcache(dbo, table)
strategy = "memcache"
elif ASM3_PK_STRATEGY == "pseq" and dbo.dbtype == "POSTGRESQL":
nextid = _get_id_postgres_seq(dbo, table)
strategy = "pseq"
else:
raise Exception("No valid PK strategy found")
if dbo.has_asm2_pk_table:
_get_id_set_asm2_primarykey(dbo, table, nextid + 1)
strategy += " asm2pk"
al.debug("get_id: %s -> %d (%s)" % (table, nextid, strategy), "db.get_id", dbo)
return nextid
def get_id(dbo, table):
"""
Returns the next ID in sequence for a table.
Will use memcache for pk generation if the CACHE_PRIMARY_KEYS option is on
and will set the cache first value if not set.
If the database has an ASM2 primary key table, it will be updated
with the next pk value.
"""
strategy = ""
nextid = 0
if ASM3_PK_STRATEGY == "max" or dbo.has_asm2_pk_table:
nextid = _get_id_max(dbo, table)
strategy = "max"
elif ASM3_PK_STRATEGY == "memcache" and cache.available():
nextid = _get_id_memcache(dbo, table)
strategy = "memcache"
elif ASM3_PK_STRATEGY == "pseq" and dbo.dbtype == "POSTGRESQL":
nextid = _get_id_postgres_seq(dbo, table)
strategy = "pseq"
else:
raise Exception("No valid PK strategy found")
if dbo.has_asm2_pk_table:
_get_id_set_asm2_primarykey(dbo, table, nextid + 1)
strategy += " asm2pk"
al.debug("get_id: %s -> %d (%s)" % (table, nextid, strategy), "db.get_id", dbo)
return nextid
def query_cache(dbo, sql, age = 60):
"""
Runs the query given and caches the result
for age seconds. If there's already a valid cached
entry for the query, returns the cached result
instead.
If CACHE_COMMON_QUERIES is set to false, just runs the query
without doing any caching and is equivalent to db.query()
"""
if not CACHE_COMMON_QUERIES or not cache.available(): return query(dbo, sql)
cache_key = "%s:%s:%s" % (dbo.alias, dbo.database, sql.replace(" ", "_"))
m = hashlib.md5()
m.update(cache_key)
cache_key = "q:%s" % m.hexdigest()
results = cache.get(cache_key)
if results is not None:
return results
results = query(dbo, sql)
cache.put(cache_key, results, age)
return results
def query_cache(dbo, sql, age=60):
"""
Runs the query given and caches the result
for age seconds. If there's already a valid cached
entry for the query, returns the cached result
instead.
If CACHE_COMMON_QUERIES is set to false, just runs the query
without doing any caching and is equivalent to db.query()
"""
if not CACHE_COMMON_QUERIES or not cache.available():
return query(dbo, sql)
cache_key = "%s:%s:%s" % (dbo.alias, dbo.database, sql.replace(" ", "_"))
m = hashlib.md5()
m.update(cache_key)
cache_key = "q:%s" % m.hexdigest()
results = cache.get(cache_key)
if results is not None:
return results
results = query(dbo, sql)
cache.put(cache_key, results, age)
return results
def get_lat_long(dbo, address, town, county, postcode, country=None):
"""
Looks up a latitude and longitude from an address using GEOCODE_URL
and returns them as lat,long,(first 3 chars of address)
Returns None if no results were found.
NB: dbo is only used for contextual reference in logging, no database
calls are made by any of this code.
"""
if address.strip() == "":
return None
try:
# Synchronise this process to a single thread to prevent
# abusing our geo provider and concurrent requests for the
# same address when opening an animal with the same
# original/brought in by owner, etc.
lat_long_lock.acquire()
url = ""
if country is None:
country = LOCALE_COUNTRY_NAME_MAP[dbo.locale]
if BULK_GEO_PROVIDER == "cloudmade":
q = normalise_cloudmade(address, town, county, postcode, country)
url = BULK_GEO_CLOUDMADE_URL.replace(
"{key}", BULK_GEO_PROVIDER_KEY).replace("{q}", q)
elif BULK_GEO_PROVIDER == "nominatim":
q = normalise_nominatim(address, town, county, postcode, country)
url = BULK_GEO_NOMINATIM_URL.replace("{q}", q)
elif BULK_GEO_PROVIDER == "smcom":
q = normalise_nominatim(address, town, county, postcode, country)
url = BULK_GEO_SMCOM_URL.replace("{q}", q)
elif BULK_GEO_PROVIDER == "google":
q = normalise_google(address, town, county, postcode, country)
url = BULK_GEO_GOOGLE_URL.replace("{q}", q)
else:
al.error("unrecognised geo provider: %s" % BULK_GEO_PROVIDER,
"geo.get_lat_long", dbo)
al.debug("looking up geocode for address: %s" % q, "geo.get_lat_long",
dbo)
key = "nom:" + q
if cache.available():
v = cache.get(key)
if v is not None:
al.debug("cache hit for address: %s = %s" % (q, v),
"geo.get_lat_long", dbo)
return v
jr = urllib2.urlopen(url, timeout=BULK_GEO_LOOKUP_TIMEOUT).read()
j = json.loads(jr)
latlon = None
if BULK_GEO_PROVIDER == "cloudmade":
latlon = parse_cloudmade(dbo, jr, j, q)
elif BULK_GEO_PROVIDER == "nominatim":
latlon = parse_nominatim(dbo, jr, j, q)
elif BULK_GEO_PROVIDER == "smcom":
latlon = parse_nominatim(dbo, jr, j, q)
elif BULK_GEO_PROVIDER == "google":
latlon = parse_google(dbo, jr, j, q)
if BULK_GEO_SLEEP_AFTER > 0:
time.sleep(BULK_GEO_SLEEP_AFTER)
return latlon
except Exception, err:
al.error(str(err), "geo.get_lat_long", dbo)
return None
