def update_user_activity(dbo, user, timenow = True):
"""
If timenow is True, updates this user's last activity time to now.
If timenow is False, removes this user from the active list.
"""
if dbo is None or user is None: return
cachekey = "%s_activity" % dbo.database
ac = utils.nulltostr(cachemem.get(cachekey))
# Prune old activity and remove the current user
nc = []
for a in ac.split(","):
# If there are any errors reading or parsing
# the entry, skip it
try:
if a != "":
u, d = a.split("=")
# if the last seen value was more than an hour ago,
# don't bother adding that user
p = i18n.parse_date("%Y-%m-%d %H:%M:%S", d)
if i18n.subtract_hours(i18n.now(dbo.timezone), 1) > p:
continue
# Don't add the current user
if u == user:
continue
nc.append(a)
except:
continue
# Add this user with the new time
if timenow:
nc.append("%s=%s" % (user, i18n.format_date("%Y-%m-%d %H:%M:%S", i18n.now(dbo.timezone))))
cachemem.put(cachekey, ",".join(nc), 3600 * 8)
def update_user_activity(dbo, user, timenow=True):
"""
If timenow is True, updates this user's last activity time to now.
If timenow is False, removes this user from the active list.
"""
if dbo is None or user is None: return
cachekey = "%s_activity" % dbo.database
ac = utils.nulltostr(cachemem.get(cachekey))
# Prune old activity and remove the current user
nc = []
for a in ac.split(","):
# If there are any errors reading or parsing
# the entry, skip it
try:
if a != "":
u, d = a.split("=")
# if the last seen value was more than an hour ago,
# don't bother adding that user
p = i18n.parse_date("%Y-%m-%d %H:%M:%S", d)
if i18n.subtract_hours(i18n.now(dbo.timezone), 1) > p:
continue
# Don't add the current user
if u == user:
continue
nc.append(a)
except:
continue
# Add this user with the new time
if timenow:
nc.append(
"%s=%s" %
(user, i18n.format_date("%Y-%m-%d %H:%M:%S", i18n.now(
dbo.timezone))))
cachemem.put(cachekey, ",".join(nc), 3600 * 8)
def _get_id_cache(dbo, table):
cache_key = "db:%s:as:%s:tb:%s" % (dbo.database, dbo.alias, table)
nextid = cachemem.increment(cache_key)
if nextid is None:
nextid = query_int(dbo, "SELECT MAX(ID) FROM %s" % table) + 1
cachemem.put(cache_key, nextid, 600)
return nextid
def get_lat_long(dbo, address, town, county, postcode, country=None):
"""
Looks up a latitude and longitude from an address using the set geocoding provider
and returns them as lat,long,hash
If no results were found, a zero lat and long are returned so that
we know not to try and look this up again until the address hash changes.
NB: dbo is only used for contextual reference in logging and obtaining locale,
no database calls are made by any of this code.
"""
if address.strip() == "":
return None
try:
# Synchronise this process to a single thread to prevent
# abusing our geo provider
lat_long_lock.acquire()
if country is None:
country = i18n.get_country(dbo.locale)
g = None
if GEO_PROVIDER == "nominatim":
g = Nominatim(dbo, address, town, county, postcode, country)
elif GEO_PROVIDER == "google":
g = Google(dbo, address, town, county, postcode, country)
elif GEO_PROVIDER == "smcom":
g = Smcom(dbo, address, town, county, postcode, country)
else:
al.error("unrecognised geo provider: %s" % GEO_PROVIDER,
"geo.get_lat_long", dbo)
return None
# Check the cache in case we already requested this address
cachekey = "nom:" + g.q
v = cachemem.get(cachekey)
if v is not None:
al.debug("cache hit for address: %s = %s" % (cachekey, v),
"geo.get_lat_long", dbo)
return v
# Call the service to get the data
g.search()
# Parse the response to a lat/long value
latlon = g.parse()
cachemem.put(cachekey, latlon, 86400)
if GEO_SLEEP_AFTER > 0:
time.sleep(GEO_SLEEP_AFTER)
return latlon
except Exception as err:
al.error(str(err), "geo.get_lat_long", dbo)
return None
finally:
lat_long_lock.release()
def db_lock(dbo):
"""
Locks the database for updates, returns True if the lock was
successful.
"""
cache_key = "%s_db_update_lock" % dbo.database
if cachemem.get(cache_key): return False
cachemem.put(cache_key, "YES", 60 * 5)
return True
def get_lat_long(dbo, address, town, county, postcode, country = None):
"""
Looks up a latitude and longitude from an address using GEOCODE_URL
and returns them as lat,long,(first 3 chars of address)
Returns None if no results were found.
NB: dbo is only used for contextual reference in logging, no database
calls are made by any of this code.
"""
if address.strip() == "":
return None
try:
# Synchronise this process to a single thread to prevent
# abusing our geo provider and concurrent requests for the
# same address when opening an animal with the same
# original/brought in by owner, etc.
lat_long_lock.acquire()
url = ""
if country is None:
country = LOCALE_COUNTRY_NAME_MAP[dbo.locale]
if BULK_GEO_PROVIDER == "nominatim":
q = normalise_nominatim(address, town, county, postcode, country)
url = BULK_GEO_NOMINATIM_URL.replace("{q}", q)
elif BULK_GEO_PROVIDER == "google":
q = normalise_google(address, town, county, postcode, country)
url = BULK_GEO_GOOGLE_URL.replace("{q}", q)
else:
al.error("unrecognised geo provider: %s" % BULK_GEO_PROVIDER, "geo.get_lat_long", dbo)
al.debug("looking up geocode for address: %s" % q, "geo.get_lat_long", dbo)
key = "nom:" + q
v = cachemem.get(key)
if v is not None:
al.debug("cache hit for address: %s = %s" % (q, v), "geo.get_lat_long", dbo)
return v
jr = urllib2.urlopen(url, timeout = BULK_GEO_LOOKUP_TIMEOUT).read()
j = json.loads(jr)
latlon = None
if BULK_GEO_PROVIDER == "nominatim":
latlon = parse_nominatim(dbo, jr, j, q)
elif BULK_GEO_PROVIDER == "google":
latlon = parse_google(dbo, jr, j, q)
if BULK_GEO_SLEEP_AFTER > 0:
time.sleep(BULK_GEO_SLEEP_AFTER)
cachemem.put(key, latlon, 86400)
return latlon
except Exception,err:
al.error(str(err), "geo.get_lat_long", dbo)
return None
def get_map(dbo):
""" Returns a map of the config items, using a read-through cache to save database calls """
CACHE_KEY = "%s_config" % dbo.database
cmap = cachemem.get(CACHE_KEY)
if cmap is None:
rows = dbo.query("SELECT ItemName, ItemValue FROM configuration ORDER BY ItemName")
cmap = DEFAULTS.copy()
for r in rows:
cmap[r.itemname] = r.itemvalue
cachemem.put(CACHE_KEY, cmap, 3600) # one hour cache means direct database updates show up eventually
return cmap
def get_account(alias):
"""
Returns the smcom account object for alias/db
Uses a read through 48 hour cache to save unnecessary calls
"""
TTL = 86400 * 2
cachekey = "smcom_dbinfo_%s" % alias
a = cachemem.get(cachekey)
if a is None:
a = smcom_client.get_account(alias)
if a is not None and "user" in a:
cachemem.put(cachekey, a, TTL)
return a
def get_account(alias):
"""
Returns the smcom account object for alias/db
Uses a read through 48 hour cache to save unnecessary calls
"""
# Attackers have tried to overflow alias in the past, we'll never use more than 20 chars
# fail fast and save us a load of processing.
if len(alias) > 20: return None
TTL = 86400 * 2
cachekey = "smcom_dbinfo_%s" % alias
a = cachemem.get(cachekey)
if a is None:
a = smcom_client.get_account(alias)
if a is not None and "user" in a:
cachemem.put(cachekey, a, TTL)
return a
def query_cache(self, sql, params=None, age=60, limit=0, distincton=""):
"""
Runs the query given and caches the result
for age seconds. If there's already a valid cached
entry for the query, returns the cached result
instead.
If CACHE_COMMON_QUERIES is set to false, just runs the query
without doing any caching and is equivalent to Database.query()
"""
if not CACHE_COMMON_QUERIES: return self.query(sql, params=params, limit=limit)
cache_key = utils.md5_hash("%s:%s:%s" % (self.database, sql, params))
results = cachemem.get(cache_key)
if results is not None:
return results
results = self.query(sql, params=params, limit=limit, distincton=distincton)
cachemem.put(cache_key, results, age)
return results
def flood_protect(method, remoteip, ttl, message = ""):
""" Checks to see if we've had a request for method from remoteip since ttl seconds ago.
If we haven't, we record this as the last time we saw a request
from this ip address for that method. Otherwise, an error is thrown.
method: The service method we're protecting
remoteip: The ip address of the caller
ttl: The protection period (one request per ttl seconds)
"""
cache_key = "m%sr%s" % (method, str(remoteip).replace(", ", "")) # X-FORWARDED-FOR can be a list, remove commas
v = cachemem.get(cache_key)
#al.debug("method: %s, remoteip: %s, ttl: %d, cacheval: %s" % (method, remoteip, ttl, v), "service.flood_protect")
if v is None:
cachemem.put(cache_key, "x", ttl)
else:
if message == "":
message = "You have already called '%s' in the last %d seconds, please wait before trying again." % (method, ttl)
raise utils.ASMError(message)
def query_cache(dbo, sql, age = 60):
"""
Runs the query given and caches the result
for age seconds. If there's already a valid cached
entry for the query, returns the cached result
instead.
If CACHE_COMMON_QUERIES is set to false, just runs the query
without doing any caching and is equivalent to db.query()
"""
if not CACHE_COMMON_QUERIES: return query(dbo, sql)
cache_key = utils.md5_hash("%s:%s:%s" % (dbo.alias, dbo.database, sql.replace(" ", "_")))
results = cachemem.get(cache_key)
if results is not None:
return results
results = query(dbo, sql)
cachemem.put(cache_key, results, age)
return results
def flood_protect(method, remoteip, ttl, message = ""):
"""
Checks to see if we've had a request for method from
remoteip since ttl seconds ago.
If we haven't, we record this as the last time we saw a request
from this ip address for that method. Otherwise, an error is thrown.
method: The service method we're protecting
remoteip: The ip address of the caller
ttl: The protection period (one request per ttl seconds)
"""
cache_key = "m%sr%s" % (method, str(remoteip).replace(", ", "")) # X-FORWARDED-FOR can be a list, remove commas
v = cachemem.get(cache_key)
#al.debug("method: %s, remoteip: %s, ttl: %d, cacheval: %s" % (method, remoteip, ttl, v), "service.flood_protect")
if v is None:
cachemem.put(cache_key, "x", ttl)
else:
if message == "":
message = "You have already called '%s' in the last %d seconds, please wait before trying again." % (method, ttl)
raise utils.ASMError(message)
def get_lat_long(dbo, address, town, county, postcode, country=""):
"""
Looks up a latitude and longitude from an address using the set geocoding provider
and returns them as lat,long,hash
If no results were found, a zero lat and long are returned so that
we know not to try and look this up again until the address hash changes.
"""
if address.strip() == "":
return None
try:
# Synchronise this process to a single thread to prevent
# abusing our geo provider
lat_long_lock.acquire()
# Use the country passed. If no country was passed, check
# if one has been set with the shelter details in settings,
# otherwise use the country from the user's locale.
if country is None or country == "":
country = configuration.organisation_country(dbo)
if country == "": country = i18n.get_country(dbo.locale)
g = None
if GEO_PROVIDER == "nominatim":
g = Nominatim(dbo, address, town, county, postcode, country)
elif GEO_PROVIDER == "google":
g = Google(dbo, address, town, county, postcode, country)
elif GEO_PROVIDER == "smcom":
g = Smcom(dbo, address, town, county, postcode, country)
else:
al.error("unrecognised geo provider: %s" % GEO_PROVIDER,
"geo.get_lat_long", dbo)
return None
# Check the cache in case we already requested this address
cachekey = "nom:" + g.q
v = cachemem.get(cachekey)
if v is not None:
al.debug("cache hit for address: %s = %s" % (cachekey, v),
"geo.get_lat_long", dbo)
return v
# Call the service to get the data
g.search()
# Parse the response to a lat/long value
latlon = g.parse()
cachemem.put(cachekey, latlon, 86400)
if GEO_SLEEP_AFTER > 0:
time.sleep(GEO_SLEEP_AFTER)
return latlon
except Exception as err:
al.error(str(err), "geo.get_lat_long", dbo)
return None
finally:
lat_long_lock.release()
def put(dbo, k, v):
""" Store a task value for this database """
cachemem.put("%s.%s" % (dbo.database, k), v, 3600)