def test_clear_metrics_cache(test_client, complete_admin_auth_token): def clear_metrics(): return test_client.post( f'/api/v1/metrics/clear_cache/', headers=dict(Authorization=complete_admin_auth_token, Accept='application/json'), ) # Do a non-test clear metrics to flush the cache clear_metrics() # Create 4 fake metrics red.set('2_metrics_fake_metric1', '123') red.set('2_metrics_fake_metric2', '123') red.set('2_metrics_fake_metric3', '123') red.set('2_metrics_fake_metric4', '123') # Clear them for reals now response = clear_metrics() # Make sure that 4 metrics are removed from the cache assert response.json['data']['removed_entries'] == 44 # And check that they're well and truly gone from the cache! assert not red.get('2_metrics_fake_metric1') assert not red.get('2_metrics_fake_metric2') assert not red.get('2_metrics_fake_metric3') assert not red.get('2_metrics_fake_metric4')
def execute_with_partial_history_cache(metric_name, query, object_model, strategy, enable_cache=True, group_by=None, query_name=''): # enable_cache pass-thru. This is so we don't cache data when filters are active. if strategy in dumb_strategies or not enable_cache: return _handle_combinatory_strategy(query, None, strategy) if query_name: metric_name = metric_name + '_' + query_name # Redis object names if g.get('query_organisations'): ORG_STRING = get_metrics_org_string(g.query_organisations) else: ORG_STRING = get_metrics_org_string(g.active_organisation.id) CURRENT_MAX_ID = f'{ORG_STRING}_{object_model.__table__.name}_max_id' HIGHEST_ID_CACHED = f'{ORG_STRING}_{metric_name}_{group_by}_max_cached_id' CACHE_RESULT = f'{ORG_STRING}_{metric_name}_{group_by}' # Checks if provided combinatry strategy is valid if strategy not in valid_strategies: raise Exception(f'Invalid combinatory strategy {strategy} requested.') # Getting the current maximum ID in the database. Also caching it so we don't have to # get it from the DB many times in the same request current_max_id = red.get(CURRENT_MAX_ID) if not current_max_id: query_max = db.session.query(db.func.max( object_model.id)).with_session(db.session).first() try: current_max_id = query_max[0] except IndexError: current_max_id = 0 red.set(CURRENT_MAX_ID, current_max_id, 10) # Gets cache results since the last time the metrics were fetched highest_id_in_cache = int(red.get(HIGHEST_ID_CACHED) or 0) cache_result = _load_cache(CACHE_RESULT) # If there's no cache (either it's a new attribute we're tracking, or the cache is corrupted) # then we should pull results starting at id=0 if cache_result: filtered_query = query.filter(object_model.id > highest_id_in_cache) else: filtered_query = query #Combines results result = _handle_combinatory_strategy(filtered_query, cache_result, strategy) # Updates the cache with new data _store_cache(CACHE_RESULT, result) red.set(HIGHEST_ID_CACHED, current_max_id, config.METRICS_CACHE_TIMEOUT) return result
def rate_limit(key, rate): """Function to impose ratelimits on an API. Pass it a key, and it'll only allow that key `rate` times per hour key: Key on which to rate limit (like a username) rate: Number of allowed requests per hour Returns True if rate limited """ # Don't ratelimit our unit tests! if current_app.config['IS_TEST']: return False # Check if the key has been tried recently if red.exists(key): attempts = int(red.get(key)) if attempts > rate: # If rate limited, return how long (minutes) until you can try again ttl = int(red.ttl(key) / 60) return ttl red.set(key, attempts + 1, keepttl=True) return False # Add key to redis to start tracking rates red.setex( key, 3600, # 1 Hour value=1) return False
def _set_user_gps_from_location(user_id: int, location: str): """ Wrapped version for testing """ # Try load location from redis cache to avoid hitting OSM too much cached_tuple_string = red.get(redis_location_key(location)) if cached_tuple_string: gps_tuple = json.loads(cached_tuple_string) else: gps_tuple = osm_location_to_gps_lookup(location) if not gps_tuple: logg.warning( f'GPS for location not found on OSM for user {user_id}') return red.set(redis_location_key(location), json.dumps(gps_tuple)) lat, lng = gps_tuple user = User.query.get(user_id) if not user: capture_message(f'User not found for id {user_id}') return user.lat = lat user.lng = lng db.session.commit()
def _load_cache(key): cached_object = red.get(key) if not cached_object: return None try: return pickle.loads(cached_object) except: red.delete(key) return None
def execute_with_partial_history_cache(metric_name, query, object_model, strategy, disable_cache = False): # disable_cache pass-thru. This is so we don't cache data when filters are active. if disable_cache: return _handle_combinatory_strategy(query, None, strategy) # Redis object names CURRENT_MAX_ID = f'{g.active_organisation.id}_{object_model.__table__.name}_max_id' HIGHEST_ID_CACHED = f'{g.active_organisation.id}_{metric_name}_max_cached_id' CACHE_RESULT = f'{g.active_organisation.id}_{metric_name}' # Checks if provided combinatry strategy is valid if strategy not in valid_strategies: raise Exception(f'Invalid combinatory strategy {strategy} requested.') # Getting the current maximum ID in the database. Also caching it so we don't have to # get it from the DB many times in the same request current_max_id = red.get(CURRENT_MAX_ID) if not current_max_id: current_max_id = db.session.query(db.func.max(object_model.id)).first() or (0, ) current_max_id = current_max_id[0] red.set(CURRENT_MAX_ID, current_max_id, 10) # Gets cache results since the last time the metrics were fetched highest_id_in_cache = int(red.get(HIGHEST_ID_CACHED) or 0) cache_result = _load_cache(CACHE_RESULT) # If there's no cache (either it's a new attribute we're tracking, or the cache is corrupted) # then we should pull results starting at id=0 if cache_result: filtered_query = query.filter(object_model.id > highest_id_in_cache) else: filtered_query = query #Combines results result = _handle_combinatory_strategy(filtered_query, cache_result, strategy) # Updates the cache with new data _store_cache(CACHE_RESULT, result) red.set(HIGHEST_ID_CACHED, current_max_id) return result
def _set_user_gps_from_location(user_id: int, location: str, skip_cache=False, user_obj=None): """ Wrapped version for testing :param user_id: the user ID to set the location for :param location: the user to set the location for :param skip_cache: flag to skip cache access (Will still write to cache) :param user_obj: the user object to set location for (Optional) """ user = user_obj or User.query.get(user_id) if not user: capture_message(f'User not found for id {user_id}') return # Add country to location lookup if it's not already there country = user.default_organisation.country if user.default_organisation else None query_location = location if country and location and country not in location: query_location = f'{location}, {country}' # Try load location from redis cache to avoid hitting OSM too much cached_tuple_string = None if not skip_cache: cached_tuple_string = red.get(redis_location_key(query_location)) if cached_tuple_string: gps_tuple = tuple(json.loads(cached_tuple_string)) else: gps_tuple = osm_location_to_gps_lookup( query_location) or get_location_from_peers(location) if not gps_tuple: logg.warning( f'GPS for location "{query_location}" not found on OSM or amongst peers for user {user_id}' ) return red.set(redis_location_key(query_location), json.dumps(gps_tuple)) lat, lng = gps_tuple user.lat = lat user.lng = lng db.session.commit()
def test_dataset_api(test_client, authed_sempo_admin_user): data = { "data": [ { "0": "Alf", "1": "Melmac", "2": "19027192211" }, { "0": "Alf", "1": "Tanner", "2": "19027192211" # Same phone number, should trigger update }, { "0": "Willie", "1": "Tanner", "2": "19027192222" } ], "headerPositions": { "0": "first_name", "1": "last_name", "2": "phone" }, "country": "", "saveName": "", "isVendor": False } auth = get_complete_auth_token(authed_sempo_admin_user) response = test_client.post( f"/api/v1/dataset/", headers=dict( Authorization=auth, Accept='application/json', ), json=data ) redis_id = get_job_key(authed_sempo_admin_user.id, response.json['task_uuid']) status = { 'percent_complete': 0 } status = json.loads(red.get(redis_id)) assert status['message'] == 'success' assert status['percent_complete'] == 100.0 assert status['diagnostics'] == [['User Created', 200], ['User Updated', 200], ['User Created', 200]]
def cached_funds_available(allowed_cache_age_seconds=60): """ IF refreshing cash THEN: return: [current blockchain balance] - [all transfers with blockchain state pending or unknown] save to cache: [funds available], [ID of highest transfer used in cache], [cache creation datetime] ELSE return: [funds available at last cache] - [all non-failed transfers since last cache] Max Txn ID is a simple way to determine whether txn was used in cache or not, and thus needs to be accounted for :param allowed_cache_age_seconds: how long between checking the blockchain for external funds added or removed :return: amount of funds available """ token = g.active_organisation.org_level_transfer_account.token balance_wei = bt.get_wallet_balance( g.active_organisation.org_level_transfer_account.blockchain_address, token) return token.token_amount_to_system(balance_wei) refresh_cache = False funds_available_cache = red.get('funds_available_cache') try: parsed_cache = json.loads(funds_available_cache) last_updated_datetime = datetime.datetime.fromtimestamp( float(parsed_cache['last_updated'])) earliest_allowed = datetime.datetime.utcnow() - datetime.timedelta( seconds=allowed_cache_age_seconds) if last_updated_datetime < earliest_allowed: refresh_cache = True except Exception as e: refresh_cache = True if refresh_cache: master_wallet_balance = bt.get_wallet_balance() highest_transfer_id_checked = 0 required_blockchain_statuses = ['PENDING', 'UNKNOWN'] else: cached_funds_available = parsed_cache['cached_funds_available'] highest_transfer_id_checked = parsed_cache[ 'highest_transfer_id_checked'] required_blockchain_statuses = ['PENDING', 'UNKNOWN', 'COMPLETE'] new_dibursements = (CreditTransfer.query.filter( CreditTransfer.transfer_type == TransferTypeEnum.PAYMENT).filter( CreditTransfer.transfer_subtype == TransferSubTypeEnum.DISBURSEMENT ).filter(CreditTransfer.transfer_status == TransferStatusEnum.COMPLETE).filter( CreditTransfer.id > highest_transfer_id_checked).filter( CreditTransfer.created > datetime.datetime.utcnow() - datetime.timedelta(hours=36)).all()) local_disbursement_value = 0 for disbursement in new_dibursements: status = disbursement.blockchain_status if status in required_blockchain_statuses: local_disbursement_value += disbursement.transfer_amount if refresh_cache: balance = master_wallet_balance - local_disbursement_value if len(new_dibursements) > 0: highest_transfer_id_checked = new_dibursements[-1].id else: all_transfers = CreditTransfer.query.all() if len(all_transfers) > 0: highest_transfer_id_checked = all_transfers[-1].id else: highest_transfer_id_checked = 0 cache_data = { 'cached_funds_available': balance, 'highest_transfer_id_checked': highest_transfer_id_checked, 'last_updated': datetime.datetime.utcnow().timestamp() } red.set('funds_available_cache', json.dumps(cache_data)) balance = master_wallet_balance - local_disbursement_value return balance else: balance = cached_funds_available - local_disbursement_value return balance
def get_job_result(user_id, func_uuid): key = get_job_key(user_id, func_uuid) return red.get(key)
def master_wallet_funds_available(allowed_cache_age_seconds=60): """ IF refreshing cash THEN: return: [current blockchain balance] - [all transfers with blockchain state pending or unknown] save to cache: [funds available at last cache], [ID of highest transfer used in cache], [cache creation datetime] ELSE return: [funds available at last cache] - [all non-failed transfers since cache created] :param allowed_cache_age_seconds: how long between checking the blockchain for external funds added or removed :return: amount of funds available """ refresh_cache = False funds_available_cache = red.get('funds_available_cache') try: parsed_cache = json.loads(funds_available_cache) last_updated_datetime = datetime.datetime.fromtimestamp( float(parsed_cache['last_updated'])) earliest_allowed = datetime.datetime.utcnow() - datetime.timedelta( seconds=allowed_cache_age_seconds) if last_updated_datetime < earliest_allowed: refresh_cache = True except Exception as e: refresh_cache = True if refresh_cache: blockchain_task = celery_app.signature( 'worker.celery_tasks.get_master_balance') result = blockchain_task.apply_async() try: master_wallet_balance = result.wait(timeout=6, propagate=True, interval=0.5) except Exception as e: print(e) sentry.captureException() raise BlockchainError("Blockchain Error") finally: result.forget() highest_transfer_id_checked = 0 required_blockchain_statuses = ['PENDING', 'UNKNOWN'] else: cached_funds_available = parsed_cache['cached_funds_available'] highest_transfer_id_checked = parsed_cache[ 'highest_transfer_id_checked'] required_blockchain_statuses = ['PENDING', 'UNKNOWN', 'COMPLETE'] new_dibursements = (models.CreditTransfer.query.filter( models.CreditTransfer.transfer_type == models.TransferTypeEnum.DISBURSEMENT).filter( models.CreditTransfer.transfer_status == models.TransferStatusEnum.COMPLETE).filter( models.CreditTransfer.id > highest_transfer_id_checked ).filter( models.CreditTransfer.created > datetime.datetime.utcnow() - datetime.timedelta(hours=36)).all()) local_disbursement_value = 0 for disbursement in new_dibursements: status = disbursement.blockchain_status if status in required_blockchain_statuses: local_disbursement_value += disbursement.transfer_amount if refresh_cache: balance = master_wallet_balance - local_disbursement_value if len(new_dibursements) > 0: highest_transfer_id_checked = new_dibursements[-1].id else: all_transfers = models.CreditTransfer.query.all() if len(all_transfers) > 0: highest_transfer_id_checked = all_transfers[-1].id else: highest_transfer_id_checked = 0 cache_data = { 'cached_funds_available': balance, 'highest_transfer_id_checked': highest_transfer_id_checked, 'last_updated': datetime.datetime.utcnow().timestamp() } red.set('funds_available_cache', json.dumps(cache_data)) balance = master_wallet_balance - local_disbursement_value return balance else: balance = cached_funds_available - local_disbursement_value return balance