def update_user_activity(dbo, user, timenow = True): """ If timenow is True, updates this user's last activity time to now. If timenow is False, removes this user from the active list. """ if dbo is None or user is None: return cachekey = "%s_activity" % dbo.database ac = utils.nulltostr(cachemem.get(cachekey)) # Prune old activity and remove the current user nc = [] for a in ac.split(","): # If there are any errors reading or parsing # the entry, skip it try: if a != "": u, d = a.split("=") # if the last seen value was more than an hour ago, # don't bother adding that user p = i18n.parse_date("%Y-%m-%d %H:%M:%S", d) if i18n.subtract_hours(i18n.now(dbo.timezone), 1) > p: continue # Don't add the current user if u == user: continue nc.append(a) except: continue # Add this user with the new time if timenow: nc.append("%s=%s" % (user, i18n.format_date("%Y-%m-%d %H:%M:%S", i18n.now(dbo.timezone)))) cachemem.put(cachekey, ",".join(nc), 3600 * 8)
def update_user_activity(dbo, user, timenow=True): """ If timenow is True, updates this user's last activity time to now. If timenow is False, removes this user from the active list. """ if dbo is None or user is None: return cachekey = "%s_activity" % dbo.database ac = utils.nulltostr(cachemem.get(cachekey)) # Prune old activity and remove the current user nc = [] for a in ac.split(","): # If there are any errors reading or parsing # the entry, skip it try: if a != "": u, d = a.split("=") # if the last seen value was more than an hour ago, # don't bother adding that user p = i18n.parse_date("%Y-%m-%d %H:%M:%S", d) if i18n.subtract_hours(i18n.now(dbo.timezone), 1) > p: continue # Don't add the current user if u == user: continue nc.append(a) except: continue # Add this user with the new time if timenow: nc.append( "%s=%s" % (user, i18n.format_date("%Y-%m-%d %H:%M:%S", i18n.now( dbo.timezone)))) cachemem.put(cachekey, ",".join(nc), 3600 * 8)
def _get_id_cache(dbo, table): cache_key = "db:%s:as:%s:tb:%s" % (dbo.database, dbo.alias, table) nextid = cachemem.increment(cache_key) if nextid is None: nextid = query_int(dbo, "SELECT MAX(ID) FROM %s" % table) + 1 cachemem.put(cache_key, nextid, 600) return nextid
def get_lat_long(dbo, address, town, county, postcode, country=None): """ Looks up a latitude and longitude from an address using the set geocoding provider and returns them as lat,long,hash If no results were found, a zero lat and long are returned so that we know not to try and look this up again until the address hash changes. NB: dbo is only used for contextual reference in logging and obtaining locale, no database calls are made by any of this code. """ if address.strip() == "": return None try: # Synchronise this process to a single thread to prevent # abusing our geo provider lat_long_lock.acquire() if country is None: country = i18n.get_country(dbo.locale) g = None if GEO_PROVIDER == "nominatim": g = Nominatim(dbo, address, town, county, postcode, country) elif GEO_PROVIDER == "google": g = Google(dbo, address, town, county, postcode, country) elif GEO_PROVIDER == "smcom": g = Smcom(dbo, address, town, county, postcode, country) else: al.error("unrecognised geo provider: %s" % GEO_PROVIDER, "geo.get_lat_long", dbo) return None # Check the cache in case we already requested this address cachekey = "nom:" + g.q v = cachemem.get(cachekey) if v is not None: al.debug("cache hit for address: %s = %s" % (cachekey, v), "geo.get_lat_long", dbo) return v # Call the service to get the data g.search() # Parse the response to a lat/long value latlon = g.parse() cachemem.put(cachekey, latlon, 86400) if GEO_SLEEP_AFTER > 0: time.sleep(GEO_SLEEP_AFTER) return latlon except Exception as err: al.error(str(err), "geo.get_lat_long", dbo) return None finally: lat_long_lock.release()
def db_lock(dbo): """ Locks the database for updates, returns True if the lock was successful. """ cache_key = "%s_db_update_lock" % dbo.database if cachemem.get(cache_key): return False cachemem.put(cache_key, "YES", 60 * 5) return True
def get_lat_long(dbo, address, town, county, postcode, country = None): """ Looks up a latitude and longitude from an address using GEOCODE_URL and returns them as lat,long,(first 3 chars of address) Returns None if no results were found. NB: dbo is only used for contextual reference in logging, no database calls are made by any of this code. """ if address.strip() == "": return None try: # Synchronise this process to a single thread to prevent # abusing our geo provider and concurrent requests for the # same address when opening an animal with the same # original/brought in by owner, etc. lat_long_lock.acquire() url = "" if country is None: country = LOCALE_COUNTRY_NAME_MAP[dbo.locale] if BULK_GEO_PROVIDER == "nominatim": q = normalise_nominatim(address, town, county, postcode, country) url = BULK_GEO_NOMINATIM_URL.replace("{q}", q) elif BULK_GEO_PROVIDER == "google": q = normalise_google(address, town, county, postcode, country) url = BULK_GEO_GOOGLE_URL.replace("{q}", q) else: al.error("unrecognised geo provider: %s" % BULK_GEO_PROVIDER, "geo.get_lat_long", dbo) al.debug("looking up geocode for address: %s" % q, "geo.get_lat_long", dbo) key = "nom:" + q v = cachemem.get(key) if v is not None: al.debug("cache hit for address: %s = %s" % (q, v), "geo.get_lat_long", dbo) return v jr = urllib2.urlopen(url, timeout = BULK_GEO_LOOKUP_TIMEOUT).read() j = json.loads(jr) latlon = None if BULK_GEO_PROVIDER == "nominatim": latlon = parse_nominatim(dbo, jr, j, q) elif BULK_GEO_PROVIDER == "google": latlon = parse_google(dbo, jr, j, q) if BULK_GEO_SLEEP_AFTER > 0: time.sleep(BULK_GEO_SLEEP_AFTER) cachemem.put(key, latlon, 86400) return latlon except Exception,err: al.error(str(err), "geo.get_lat_long", dbo) return None
def get_map(dbo): """ Returns a map of the config items, using a read-through cache to save database calls """ CACHE_KEY = "%s_config" % dbo.database cmap = cachemem.get(CACHE_KEY) if cmap is None: rows = dbo.query("SELECT ItemName, ItemValue FROM configuration ORDER BY ItemName") cmap = DEFAULTS.copy() for r in rows: cmap[r.itemname] = r.itemvalue cachemem.put(CACHE_KEY, cmap, 3600) # one hour cache means direct database updates show up eventually return cmap
def get_account(alias): """ Returns the smcom account object for alias/db Uses a read through 48 hour cache to save unnecessary calls """ TTL = 86400 * 2 cachekey = "smcom_dbinfo_%s" % alias a = cachemem.get(cachekey) if a is None: a = smcom_client.get_account(alias) if a is not None and "user" in a: cachemem.put(cachekey, a, TTL) return a
def get_account(alias): """ Returns the smcom account object for alias/db Uses a read through 48 hour cache to save unnecessary calls """ # Attackers have tried to overflow alias in the past, we'll never use more than 20 chars # fail fast and save us a load of processing. if len(alias) > 20: return None TTL = 86400 * 2 cachekey = "smcom_dbinfo_%s" % alias a = cachemem.get(cachekey) if a is None: a = smcom_client.get_account(alias) if a is not None and "user" in a: cachemem.put(cachekey, a, TTL) return a
def query_cache(self, sql, params=None, age=60, limit=0, distincton=""): """ Runs the query given and caches the result for age seconds. If there's already a valid cached entry for the query, returns the cached result instead. If CACHE_COMMON_QUERIES is set to false, just runs the query without doing any caching and is equivalent to Database.query() """ if not CACHE_COMMON_QUERIES: return self.query(sql, params=params, limit=limit) cache_key = utils.md5_hash("%s:%s:%s" % (self.database, sql, params)) results = cachemem.get(cache_key) if results is not None: return results results = self.query(sql, params=params, limit=limit, distincton=distincton) cachemem.put(cache_key, results, age) return results
def flood_protect(method, remoteip, ttl, message = ""): """ Checks to see if we've had a request for method from remoteip since ttl seconds ago. If we haven't, we record this as the last time we saw a request from this ip address for that method. Otherwise, an error is thrown. method: The service method we're protecting remoteip: The ip address of the caller ttl: The protection period (one request per ttl seconds) """ cache_key = "m%sr%s" % (method, str(remoteip).replace(", ", "")) # X-FORWARDED-FOR can be a list, remove commas v = cachemem.get(cache_key) #al.debug("method: %s, remoteip: %s, ttl: %d, cacheval: %s" % (method, remoteip, ttl, v), "service.flood_protect") if v is None: cachemem.put(cache_key, "x", ttl) else: if message == "": message = "You have already called '%s' in the last %d seconds, please wait before trying again." % (method, ttl) raise utils.ASMError(message)
def query_cache(dbo, sql, age = 60): """ Runs the query given and caches the result for age seconds. If there's already a valid cached entry for the query, returns the cached result instead. If CACHE_COMMON_QUERIES is set to false, just runs the query without doing any caching and is equivalent to db.query() """ if not CACHE_COMMON_QUERIES: return query(dbo, sql) cache_key = utils.md5_hash("%s:%s:%s" % (dbo.alias, dbo.database, sql.replace(" ", "_"))) results = cachemem.get(cache_key) if results is not None: return results results = query(dbo, sql) cachemem.put(cache_key, results, age) return results
def flood_protect(method, remoteip, ttl, message = ""): """ Checks to see if we've had a request for method from remoteip since ttl seconds ago. If we haven't, we record this as the last time we saw a request from this ip address for that method. Otherwise, an error is thrown. method: The service method we're protecting remoteip: The ip address of the caller ttl: The protection period (one request per ttl seconds) """ cache_key = "m%sr%s" % (method, str(remoteip).replace(", ", "")) # X-FORWARDED-FOR can be a list, remove commas v = cachemem.get(cache_key) #al.debug("method: %s, remoteip: %s, ttl: %d, cacheval: %s" % (method, remoteip, ttl, v), "service.flood_protect") if v is None: cachemem.put(cache_key, "x", ttl) else: if message == "": message = "You have already called '%s' in the last %d seconds, please wait before trying again." % (method, ttl) raise utils.ASMError(message)
def get_lat_long(dbo, address, town, county, postcode, country=""): """ Looks up a latitude and longitude from an address using the set geocoding provider and returns them as lat,long,hash If no results were found, a zero lat and long are returned so that we know not to try and look this up again until the address hash changes. """ if address.strip() == "": return None try: # Synchronise this process to a single thread to prevent # abusing our geo provider lat_long_lock.acquire() # Use the country passed. If no country was passed, check # if one has been set with the shelter details in settings, # otherwise use the country from the user's locale. if country is None or country == "": country = configuration.organisation_country(dbo) if country == "": country = i18n.get_country(dbo.locale) g = None if GEO_PROVIDER == "nominatim": g = Nominatim(dbo, address, town, county, postcode, country) elif GEO_PROVIDER == "google": g = Google(dbo, address, town, county, postcode, country) elif GEO_PROVIDER == "smcom": g = Smcom(dbo, address, town, county, postcode, country) else: al.error("unrecognised geo provider: %s" % GEO_PROVIDER, "geo.get_lat_long", dbo) return None # Check the cache in case we already requested this address cachekey = "nom:" + g.q v = cachemem.get(cachekey) if v is not None: al.debug("cache hit for address: %s = %s" % (cachekey, v), "geo.get_lat_long", dbo) return v # Call the service to get the data g.search() # Parse the response to a lat/long value latlon = g.parse() cachemem.put(cachekey, latlon, 86400) if GEO_SLEEP_AFTER > 0: time.sleep(GEO_SLEEP_AFTER) return latlon except Exception as err: al.error(str(err), "geo.get_lat_long", dbo) return None finally: lat_long_lock.release()
def put(dbo, k, v): """ Store a task value for this database """ cachemem.put("%s.%s" % (dbo.database, k), v, 3600)