async def test_locmem_cache_uses_net_loc_for_separaing_namespaces(): cache = Cache("locmem://") other_cache = Cache("locmem://other") await cache.connect() await other_cache.connect() await cache.set("test", "Ok!") assert await other_cache.get("test") is None assert await cache.get("test") == "Ok!"
def update_keepalive(params: inputs.Inputs, keepalive_state: KeepaliveState, cache: Cache): ''' Update the keepalive state in cache. Also check if the current backing job owns the keepalive. If not, exit ''' try: cache_keys = keepalive_state.cache_keys exit_if_necessary(keepalive_state, cache) keepalive_state.last_keepalive_ms = utils.millitime() cache.set(cache_keys.keepalive, pickle.dumps(keepalive_state)) except Exception as e: print("update_keepalive: exception", e, traceback.format_exc())
async def test_cache_response() -> None: cache = Cache("locmem://null", ttl=2 * 60) spy = CacheSpy(PlainTextResponse("Hello, world!")) app = CacheMiddleware(spy, cache=cache) client = httpx.AsyncClient(app=app, base_url="http://testserver") async with cache, client: assert spy.misses == 0 r = await client.get("/") assert r.status_code == 200 assert r.text == "Hello, world!" assert spy.misses == 1 assert "Expires" in r.headers expires_fmt = "%a, %d %b %Y %H:%M:%S GMT" expires = dt.datetime.strptime(r.headers["Expires"], expires_fmt) delta: dt.timedelta = expires - dt.datetime.utcnow() assert delta.total_seconds() == pytest.approx(120, rel=1e-2) assert "Cache-Control" in r.headers assert r.headers["Cache-Control"] == "max-age=120" r1 = await client.get("/") assert spy.misses == 1 assert ComparableHTTPXResponse(r1) == r r2 = await client.get("/") assert spy.misses == 1 assert ComparableHTTPXResponse(r2) == r
async def test_cookies_in_response_and_cookieless_request() -> None: """ Responses that set cookies shouldn't be cached if the request doesn't have cookies. """ cache = Cache("locmem://null") async def app(scope: Scope, receive: Receive, send: Send) -> None: response = PlainTextResponse("Hello, world!") response.set_cookie("session_id", "1234") await response(scope, receive, send) spy = CacheSpy(app) app = CacheMiddleware(spy, cache=cache) client = httpx.AsyncClient(app=app, base_url="http://testserver") async with cache, client: r = await client.get("/") assert r.status_code == 200 assert r.text == "Hello, world!" assert spy.misses == 1 r = await client.get("/") assert r.status_code == 200 assert r.text == "Hello, world!" assert spy.misses == 2
def generate_cache_key( request: Request, method: str, varying_headers: typing.List[str], cache: Cache, ) -> str: """ Return a cache key generated from the request full URL and varying response headers. Note that the given `method` may be different from that of the request, e.g. because we're trying to find a response cached from a previous GET request while this one is a HEAD request. (This is OK because web servers will strip content from responses to a HEAD request before sending them on the wire.) """ assert method in CACHABLE_METHODS ctx = hashlib.md5() for header in varying_headers: value = request.headers.get(header) if value is not None: ctx.update(value.encode()) absolute_url = str(request.url) url = hashlib.md5(absolute_url.encode("ascii")) return cache.make_key( f"cache_page.{method}.{url.hexdigest()}.{ctx.hexdigest()}")
async def cache(): obj = Cache("redis://localhost:6379/1") await obj.connect() await obj.clear() yield obj await obj.clear() await obj.disconnect()
async def test_streaming_response() -> None: """Streaming responses should not be cached.""" cache = Cache("locmem://null") async def body() -> typing.AsyncIterator[str]: yield "Hello, " yield "world!" async def app(scope: Scope, receive: Receive, send: Send) -> None: response = StreamingResponse(body()) await response(scope, receive, send) spy = CacheSpy(app) app = CacheMiddleware(spy, cache=cache) client = httpx.AsyncClient(app=app, base_url="http://testserver") async with cache, client: assert spy.misses == 0 r = await client.get("/") assert r.status_code == 200 assert r.text == "Hello, world!" assert spy.misses == 1 r = await client.get("/") assert r.status_code == 200 assert r.text == "Hello, world!" assert spy.misses == 2
async def test_decorator_raw_asgi() -> None: cache = Cache("locmem://null", ttl=2 * 60) @cached(cache) async def app(scope: Scope, receive: Receive, send: Send) -> None: response = PlainTextResponse("Hello, world!") await response(scope, receive, send) spy = app.app = CacheSpy(app.app) client = httpx.AsyncClient(app=app, base_url="http://testserver") async with cache, client: assert spy.misses == 0 r = await client.get("/") assert r.status_code == 200 assert r.text == "Hello, world!" assert "Expires" in r.headers assert "Cache-Control" in r.headers assert spy.misses == 1 r = await client.get("/") assert r.status_code == 200 assert r.text == "Hello, world!" assert "Expires" in r.headers assert "Cache-Control" in r.headers assert spy.misses == 1
async def test_use_cached_head_response_on_get() -> None: """ Making a HEAD request should use the cached response for future GET requests. """ cache = Cache("locmem://null") spy = CacheSpy(PlainTextResponse("Hello, world!")) app = CacheMiddleware(spy, cache=cache) client = httpx.AsyncClient(app=app, base_url="http://testserver") async with cache, client: assert spy.misses == 0 r = await client.head("/") assert not r.text assert r.status_code == 200 assert "Expires" in r.headers assert "Cache-Control" in r.headers assert spy.misses == 1 r1 = await client.get("/") assert r1.text == "Hello, world!" assert r1.status_code == 200 assert "Expires" in r.headers assert "Cache-Control" in r.headers assert spy.misses == 1
async def test_not_http() -> None: async def app(scope: Scope, receive: Receive, send: Send) -> None: assert scope["type"] == "lifespan" cache = Cache("locmem://null") app = CacheMiddleware(app, cache=cache) await app({"type": "lifespan"}, mock_receive, mock_send)
async def test_decorate_starlette_view() -> None: cache = Cache("locmem://null", ttl=2 * 60) with pytest.raises(ValueError): @cached(cache) async def home(request: Request) -> Response: ... # pragma: no cover
async def test_cache_can_be_used_as_context_manager(): async with Cache("locmem://") as cache: await cache.set("test", "Ok!") assert await cache.get("test") == "Ok!" assert await cache.get_or_set('test2', _testing_coroutine('arg', test1='kwarg')) == "Ok!" assert await cache(_testing_coroutine('arg', test1='kwarg')) == 'Ok!'
def generate_varying_headers_cache_key(request: Request, cache: Cache) -> str: """ Return a cache key generated from the requested absolute URL, suitable for associating varying headers to a requested URL. """ url = request.url.path url_hash = hashlib.md5(url.encode("ascii")) return cache.make_key(f"varying_headers.{url_hash.hexdigest()}")
async def test_vary() -> None: """ Sending different values for request headers registered as varying should result in different cache entries. """ cache = Cache("locmem://null") async def gzippable_app(scope: Scope, receive: Receive, send: Send) -> None: headers = Headers(scope=scope) if "gzip" in headers.getlist("accept-encoding"): body = gzip.compress(b"Hello, world!") response = PlainTextResponse( content=body, headers={ "Content-Encoding": "gzip", "Content-Length": str(len(body)) }, ) else: response = PlainTextResponse("Hello, world!") response.headers["Vary"] = "Accept-Encoding" await response(scope, receive, send) spy = CacheSpy(gzippable_app) app = CacheMiddleware(spy, cache=cache) client = httpx.AsyncClient(app=app, base_url="http://testserver") async with cache, client: r = await client.get("/", headers={"accept-encoding": "gzip"}) assert spy.misses == 1 assert r.status_code == 200 assert r.text == "Hello, world!" assert r.headers["content-encoding"] == "gzip" assert "Expires" in r.headers assert "Cache-Control" in r.headers # Different "Accept-Encoding" header => the cached result # for "Accept-Encoding: gzip" should not be used. r1 = await client.get("/", headers={"accept-encoding": "identity"}) assert spy.misses == 2 assert r1.status_code == 200 assert r1.text == "Hello, world!" assert "Expires" in r.headers assert "Cache-Control" in r.headers # This "Accept-Encoding" header has already been seen => we should # get a cached response. r2 = await client.get("/", headers={"accept-encoding": "gzip"}) assert spy.misses == 2 assert r.status_code == 200 assert r.text == "Hello, world!" assert r2.headers["Content-Encoding"] == "gzip" assert "Expires" in r.headers assert "Cache-Control" in r.headers
def keepalive_fn(scheduler: sched.scheduler, params: inputs.Inputs, context: LambdaContext, keepalive_state: KeepaliveState, cache: Cache): ''' Each iteration of keepalive_thread runs this code. Add the next iteration of keepalive before exiting to continue the keepalive thread. Otherwise keepalives will stop ''' try: update_keepalive(params, keepalive_state, cache) keepalive_fn.num_keepalives += 1 if keepalive_fn.num_keepalives % defaults.KEEPALIVE_PRINT_EVERY == 0: print("keepalive_fn: keepalive #{}: state={}".format( keepalive_fn.num_keepalives, keepalive_state)) if context.invoked_function_arn and context.get_remaining_time_in_millis( ) < defaults.RETRIGGER_BEFORE_EXPIRY_MS: # if invoked as lambda (not CLI), then retrigger backing job if this instance of it will expire soon cache_keys = keepalive_state.cache_keys lastaccess_ms = int(cache.get(cache_keys.lastaccess)) lastaccess_age_ms = utils.millitime() - lastaccess_ms if lastaccess_age_ms > (defaults.BACKING_JOB_LIFETIME_MS * 0.9): # There were no recent calls to fetch the data produced by this backing job. No need to re-issue print( "Exiting backing job by ending keepalive thread. lastaccess_age_ms = ", lastaccess_age_ms) return False if not params.is_streaming(): ''' Fixed time-range jobs need not be reissued ''' print( "keepalive_fn: backing job won't be restarted because it is not a streaming job", params) return False # Restart this job again in another lambda invocation. # Before doing that, don't keepalive for a while to make it stale. Otherwise the new invocation # will assume there is another backing job already running and will auto-exit print( "keepalive_fn: backing job needs to be restarted. lastaccess_age_ms =", lastaccess_age_ms) time.sleep(defaults.KEEPALIVE_INTERVAL_SEC * defaults.KEEPALIVE_EXPIRY_MULTIPLE) start_backing_job_if_necessary(params, context, keepalive_state, cache) print( "keepalive_fn: exiting current backing job after re-issuing a new one" ) return False except Exception as e: print("keepalive_fn: exception", e, traceback.format_exc()) # schedule the next iteration of keepalive thread scheduler.enter(defaults.KEEPALIVE_INTERVAL_SEC, 1, keepalive_fn, argument=(scheduler, params, context, keepalive_state, cache))
async def test_duplicate_caching() -> None: cache = Cache("locmem://default") special_cache = Cache("locmem://special") class DuplicateCache(HTTPEndpoint): pass app = Starlette( routes=[ Route("/duplicate_cache", CacheMiddleware(DuplicateCache, cache=special_cache)) ], middleware=[Middleware(CacheMiddleware, cache=cache)], ) client = httpx.AsyncClient(app=app, base_url="http://testserver") async with cache, special_cache, client: with pytest.raises(DuplicateCaching): await client.get("/duplicate_cache")
async def test_cache_not_connected() -> None: cache = Cache("locmem://null", ttl=2 * 60) app = CacheMiddleware(PlainTextResponse("Hello, world!"), cache=cache) client = httpx.AsyncClient(app=app, base_url="http://testserver") async with client: with pytest.raises(CacheNotConnected) as ctx: await client.get("/") exc = ctx.value assert exc.cache is cache assert str(cache.url) in str(exc)
async def test_logs_trace(capsys: typing.Any) -> None: cache = Cache("locmem://null", ttl=2 * 60) app = CacheMiddleware(PlainTextResponse("Hello, world!"), cache=cache) client = httpx.AsyncClient(app=app, base_url="http://testserver") async with cache, client: with override_log_level("trace"): await client.get("/") stderr = capsys.readouterr().err assert "cache_lookup MISS" in stderr assert "get_from_cache request.url='http://testserver/" in stderr
def metadata_consumer_thread_fn(metadata_queue: Queue, keepalive_state: KeepaliveState, cache: Cache): ''' Thread that consumes metadata messages, updates full metadata in cache ''' print("metadata_consumer_thread_fn: started") try: cache_keys = keepalive_state.cache_keys metadata = {} while (True): try: publish = False while (not metadata_queue.empty()): msg = metadata_queue.get() metadata[msg.tsid] = msg.properties publish = True if publish: cache.set(cache_keys.metadata, metadata) except Exception as e: print("metadata_consumer_thread_fn: exception", e, traceback.format_exc()) time.sleep(1.0) finally: print("metadata_consumer_thread_fn: ended")
async def test_decorator_starlette_endpoint() -> None: cache = Cache("locmem://null", ttl=2 * 60) @cached(cache) class CachedHome(HTTPEndpoint): async def get(self, request: Request) -> Response: return PlainTextResponse("Hello, world!") class UncachedUsers(HTTPEndpoint): async def get(self, request: Request) -> Response: return PlainTextResponse("Hello, users!") assert isinstance(CachedHome, CacheMiddleware) spy = CachedHome.app = CacheSpy(CachedHome.app) users_spy = CacheSpy(UncachedUsers) app = Starlette(routes=[Route("/", CachedHome), Route("/users", users_spy)]) client = httpx.AsyncClient(app=app, base_url="http://testserver") async with cache, client: assert spy.misses == 0 r = await client.get("/") assert r.status_code == 200 assert r.text == "Hello, world!" assert "Expires" in r.headers assert "Cache-Control" in r.headers assert spy.misses == 1 r = await client.get("/") assert r.status_code == 200 assert r.text == "Hello, world!" assert "Expires" in r.headers assert "Cache-Control" in r.headers assert spy.misses == 1 assert users_spy.misses == 0 r = await client.get("/users") assert r.status_code == 200 assert r.text == "Hello, users!" assert "Expires" not in r.headers assert "Cache-Control" not in r.headers assert users_spy.misses == 1 r = await client.get("/users") assert r.status_code == 200 assert r.text == "Hello, users!" assert "Expires" not in r.headers assert "Cache-Control" not in r.headers assert users_spy.misses == 2
async def test_non_cachable_zero_ttl(cache: Cache) -> None: """ We shouldn't bother caching if the cache TTL is zero. """ cache.ttl = 0 scope: Scope = { "type": "http", "method": "GET", "path": "/path", "headers": [], } request = Request(scope) response = PlainTextResponse("Hello, world!") with pytest.raises(ResponseNotCachable): await store_in_cache(response, request=request, cache=cache)
async def test_logs_debug(capsys: typing.Any) -> None: cache = Cache("locmem://null", ttl=2 * 60) app = CacheMiddleware(PlainTextResponse("Hello, world!"), cache=cache) client = httpx.AsyncClient(app=app, base_url="http://testserver") async with cache, client: with override_log_level("debug"): await client.get("/") await client.get("/") stderr = capsys.readouterr().err miss_line, store_line, hit_line, *_ = stderr.split("\n") assert "cache_lookup MISS" in miss_line assert "store_in_cache max_age=120" in store_line assert "cache_lookup HIT" in hit_line assert "get_from_cache request.url='http://testserver/" not in stderr
async def test_not_200_ok(status_code: int) -> None: """Responses that don't have status code 200 should not be cached.""" cache = Cache("locmem://null") spy = CacheSpy(PlainTextResponse("Hello, world!", status_code=status_code)) app = CacheMiddleware(spy, cache=cache) client = httpx.AsyncClient(app=app, base_url="http://testserver") async with cache, client: r = await client.get("/") assert r.status_code == status_code assert r.text == "Hello, world!" assert "Expires" not in r.headers assert "Cache-Control" not in r.headers assert spy.misses == 1 r1 = await client.get("/") assert ComparableHTTPXResponse(r1) == r assert spy.misses == 2
def exit_if_necessary(keepalive_state: KeepaliveState, cache: Cache): ''' if backing job ever discovers that another instance of the same thing is currently running and owns the keepalive key in cache, then it exits ''' cache_keys = keepalive_state.cache_keys try: cached_state: KeepaliveState = pickle.loads( cache.get(cache_keys.keepalive)) if cached_state.id != keepalive_state.id: expiry_ms = defaults.KEEPALIVE_EXPIRY_MULTIPLE * defaults.KEEPALIVE_INTERVAL_SEC * 1000 if utils.millitime() - cached_state.last_keepalive_ms < expiry_ms: # Another backing job is running, and it has published a keepalive recently print( "exit_if_necessary: exiting because another instance already running", cached_state.id, time.ctime(cached_state.last_keepalive_ms / 1000)) os._exit(1) except Exception as e: print("exit_if_necessary: failed to read keepalive from cache", e)
async def test_non_cachable_request() -> None: cache = Cache("locmem://null") spy = CacheSpy(PlainTextResponse("Hello, world!")) app = CacheMiddleware(spy, cache=cache) client = httpx.AsyncClient(app=app, base_url="http://testserver") async with cache, client: assert spy.misses == 0 r = await client.post("/") assert r.status_code == 200 assert r.text == "Hello, world!" assert "Expires" not in r.headers assert "Cache-Control" not in r.headers assert spy.misses == 1 r1 = await client.post("/") assert ComparableHTTPXResponse(r1) == r assert spy.misses == 2
def setup(project_settings: str = None, database: bool = False) -> GraphQL: """Load Turbulette applications and return the GraphQL route.""" project_settings_module = (get_project_settings_by_env() if not project_settings else import_module(project_settings)) # The database connection has to be initialized before the LazySettings object to be setup # so we have to connect to the database before the registry to be setup if database: get_gino_instance() registry = Registry(project_settings_module=project_settings_module) conf.registry.__setup__(registry) schema = registry.setup() # At this point, settings are now available through `settings` from `turbulette.conf` module settings = conf.settings # Now that the database connection is established, we can use `settings` cache.__setup__(Cache(settings.CACHE)) extensions: List[Type[Extension]] = [PolicyExtension] for ext in settings.ARIADNE_EXTENSIONS: module_class = ext.rsplit(".", 1) extensions.append( getattr( import_module(module_class[0]), module_class[1], )) graphql_route = GraphQL( schema, debug=settings.DEBUG, extensions=extensions, error_formatter=error_formatter, ) return graphql_route
def wait_for_backing_job_to_exit_batch_phase( keepalive_state: KeepaliveState, cache: Cache, cache_keys: CacheKeys, wait_until_ms: int): print("wait_for_backing_job_to_exit_batch_phase: started", cache_keys.keepalive) while not keepalive_state or not keepalive_state.in_streaming_phase: # wait for backing job to be running and advance to streaming state if utils.millitime() > wait_until_ms: raise Exception( "wait_for_backing_job_to_exit_batch_phase: timed out") print( "get_cached_result: waiting for batch phase to end. keepalive_state=", keepalive_state) time.sleep(1) try: keepalive_state: KeepaliveState = pickle.loads( cache.get(cache_keys.keepalive)) except Exception as e: print( "wait_for_backing_job_to_exit_batch_phase: failed to read keepalive from cache", cache_keys.keepalive, e) print("wait_for_backing_job_to_exit_batch_phase: backing job is ready", keepalive_state) return keepalive_state
def get_cached_result(params: inputs.Inputs, context: LambdaContext, cache: Cache): ''' Backing job is already running. So just query cached data from and return result ''' def wait_for_backing_job_to_exit_batch_phase( keepalive_state: KeepaliveState, cache: Cache, cache_keys: CacheKeys, wait_until_ms: int): print("wait_for_backing_job_to_exit_batch_phase: started", cache_keys.keepalive) while not keepalive_state or not keepalive_state.in_streaming_phase: # wait for backing job to be running and advance to streaming state if utils.millitime() > wait_until_ms: raise Exception( "wait_for_backing_job_to_exit_batch_phase: timed out") print( "get_cached_result: waiting for batch phase to end. keepalive_state=", keepalive_state) time.sleep(1) try: keepalive_state: KeepaliveState = pickle.loads( cache.get(cache_keys.keepalive)) except Exception as e: print( "wait_for_backing_job_to_exit_batch_phase: failed to read keepalive from cache", cache_keys.keepalive, e) print("wait_for_backing_job_to_exit_batch_phase: backing job is ready", keepalive_state) return keepalive_state print("get_cached_result: started") # Update 'lastaccess' timestamp in memcache to indicate the corresponding backing job's data was recently queried cache_keys: CacheKeys = CacheKeys(params.cache_key_prefix()) now_ms = params.invoke_time_ms try: cache.set(cache_keys.lastaccess, now_ms) except Exception as e: print( "get_cached_result: failed to set lastaccess cache key {}={}, {}". format(cache_keys.lastaccess, now_ms, e)) # start the backing job if one is not running, or if the backing job's keepalive timestamp is stale keepalive_state: KeepaliveState = start_backing_job_if_necessary( params, context, cache) # now that backing job is surely running, wait for it to become 'ready' - i.e. go from batch to streaming phase keepalive_state = wait_for_backing_job_to_exit_batch_phase( keepalive_state, cache, cache_keys, now_ms + defaults.API_TIMEOUT_MS) # compute which cache keys need to be fetched if not params.is_streaming(): tstart = params.absolute_ms(params.start_time_ms) tend = params.absolute_ms(params.end_time_ms) else: tend = now_ms tstart = tend - params.duration_ms() timestamps = sorted([ ts for ts in keepalive_state.data_timestamps if ts >= tstart and ts <= tend ]) data_keys = [cache_keys.data_prefix + str(ts) for ts in timestamps] # retrieve metadata and data from cache. retry if necessary metadata = cache.get(cache_keys.metadata) if len(timestamps): print( "get_cached_result: fetching {} timestamps {} - {} @ {}ms".format( len(timestamps), time.ctime(timestamps[0] / 1000), time.ctime(timestamps[-1] / 1000), keepalive_state.resolution_ms)) data = cache.multiget(data_keys) missing_keys = set(data_keys) - set(data.keys()) if (len(missing_keys)): print("get_cached_result: retrying fetch of {}/{} keys: {}".format( len(missing_keys), len(data_keys), sorted(missing_keys))) data.update(cache.multiget(list(missing_keys))) # Fill in results in results struct result = { "start_time_ms": tstart, "end_time_ms": tend, "earliest_result_ms": 0, "latest_result_ms": 0, "resolution_ms": keepalive_state.resolution_ms, "metadata": metadata, "data": {}, "missing_timestamps_ms": [] } # First fill in retrieved data tsids = set() missing_timestamps = [] for timestamp in timestamps: k = cache_keys.data_prefix + str(timestamp) if k in data.keys(): for tsid, value in data[k].items(): if not result["earliest_result_ms"]: result["earliest_result_ms"] = timestamp if timestamp > result["latest_result_ms"]: result["latest_result_ms"] = timestamp tsids.add(tsid) result["data"].setdefault(tsid, []) result["data"][tsid].append([timestamp, value]) else: missing_timestamps.append(timestamp) # Second, fill in metadata of only the relevant mts that have data remove_metadata_ids = set(metadata.keys()).difference(tsids) for tsid in remove_metadata_ids: metadata.pop(tsid) result["missing_timestamps_ms"] = missing_timestamps return result
def start_backing_job_if_necessary(params: inputs.Inputs, context: LambdaContext, cache: Cache): ''' If no backing job is running for a given signalflow program and duration, start one Returns keepalive_state from cache if active backing job is found (to prevent a duplicate cache read by callers ''' def start_backing_job_as_lambda(params: inputs.Inputs, tstart, tend, context: LambdaContext): # Start new backing job that runs as a lambda function print("start_backing_job_as_lambda: started") import boto3 lambda_client = boto3.client('lambda') lambda_client.invoke(FunctionName=context.invoked_function_arn, InvocationType='Event', Payload=json.dumps({ "program": params.program, "start_time_ms": tstart, "end_time_ms": tend, "resolution_hint_ms": params.resolution_hint_ms, "api_token": params.api_token, "api_endpoint": params.api_endpoint, "daemon": True })) def start_backing_job_as_process(params: inputs.Inputs, tstart, tend): # Start new backing job that runs as a python process print("start_backing_job_as_process: started") cmd: str = "nohup python3 {script} --program=\"{program}\" --token={token} \ --start_time_ms={tstart} --end_time_ms={tend} --resolution_hint_ms={res} --endpoint={endpoint}".format( script=__file__, program=params.program, tstart=tstart, tend=tend, res=params.resolution_hint_ms, token=params.api_token, endpoint=params.api_endpoint) cmd += " --daemon > /tmp/{}.log 2>&1 &".format( params.cache_key_prefix()) print("start_backing_job_as_process:", cmd) os.system(cmd) # begin code for start_backing_job_if_necessary() try: cache_keys = CacheKeys(params.cache_key_prefix()) print("start_backing_job_if_necessary: started", cache_keys) now_ms = utils.millitime() cached_state: KeepaliveState = pickle.loads( cache.get(cache_keys.keepalive)) keepalive_age_ms = now_ms - cached_state.last_keepalive_ms expiry_ms = defaults.KEEPALIVE_EXPIRY_MULTIPLE * defaults.KEEPALIVE_INTERVAL_SEC * 1000 if keepalive_age_ms < expiry_ms: print( "start_backing_job_if_necessary: found active backing job already running. keepalive_age_ms =", keepalive_age_ms) return cached_state print( "start_backing_job_if_necessary: found expired keepalive_age_ms =", keepalive_age_ms) cache.set(cache_keys.keepalive, None) except Exception as e: print("start_backing_job_if_necessary: no keeplive found in cache", e) tstart = params.start_time_ms tend = params.end_time_ms if not params.is_streaming(): tstart = params.absolute_ms(tstart) tend = params.absolute_ms(tend) if context.invoked_function_arn: # This backing job was invoked as a lambda. So invoke a new lambda start_backing_job_as_lambda(params, tstart, tend, context) else: start_backing_job_as_process(params, tstart, tend) return None
def data_consumer_thread_fn(params: inputs.Inputs, context: LambdaContext, data_queue: Queue, keepalive_state: KeepaliveState, cache: Cache): ''' Thread that consumes data messages from analytics job, and sticks each one individually into cache. Also detects when job moves from batch to stream phase. Unfortunately that requires 'auto-detection' where data does not arrive for close to a second :-( ''' print("data_consumer_thread_fn: started") try: cache_keys = keepalive_state.cache_keys data_to_encache = {} last_datamsg_walltime_ms = 0 while True: now_ms = utils.millitime() try: if params.is_streaming(): # remove trailing data keys that are beyond the scope of the current 'window' of a streaming job valid_timestamps = [ ts for ts in keepalive_state.data_timestamps if ts >= (now_ms - params.job_duration_ms() - keepalive_state.resolution_ms) ] keepalive_state.data_timestamps = set(valid_timestamps) msg = data_queue.get(False) last_datamsg_walltime_ms = utils.millitime() data_to_encache.setdefault(msg.logical_timestamp_ms, {}) data_to_encache[msg.logical_timestamp_ms].update(msg.data) except Exception as e: # No data found in queue. However there may be pending data from previous messages that need caching timestamps_encached = set() for timestamp, values in data_to_encache.items(): try: cache.set(cache_keys.data_prefix + str(timestamp), values) timestamps_encached.add(timestamp) except Exception as e: # Failed to set data in cache None for timestamp_encached in timestamps_encached: data_to_encache.pop(timestamp_encached) keepalive_state.data_timestamps.add(timestamp_encached) if data_to_encache: print( "data_consumer_thread_fn: will retry writing {} data keys to cache {}" .format(len(data_to_encache), list(data_to_encache))) elif not keepalive_state.in_streaming_phase: # Now that all data is successfully published, 'Auto-detect' whether we have completed batch phase # and entered stream phase. If so, update keepalive_state if last_datamsg_walltime_ms > 0 and ( now_ms - last_datamsg_walltime_ms ) > defaults.STREAM_PHASE_DETECTION_INTERVAL_MS: keepalive_state.in_streaming_phase = True print( "data_consumer_thread_fn: backing job entered stream phase after {} datapoints. now={}, last={}" .format(len(keepalive_state.data_timestamps), now_ms, last_datamsg_walltime_ms)) # start healthcheck thread now that data is flowing in threading.Thread(target=healthcheck_thread_fn, args=(params, context, keepalive_state, cache)).start() time.sleep(defaults.STREAM_PHASE_DETECTION_INTERVAL_MS / 1000 / 5) except Exception as e: print("data_consumer_thread_fn exception", e, traceback.format_exc()) finally: print("data_consumer_thread_fn: ended")