def query(event: Dict[str, Any]) -> None: """Run the query portion of a QJ""" query_config = QueryConfig() logger = Logger() logger.info(event=QJLogEvents.InitConfig, config=query_config) records = event.get("Records", []) if not records: raise Exception("No records found") if len(records) > 1: raise Exception( f"More than one record. BatchSize is probably not 1. event: {event}" ) body = records[0].get("body") if body is None: raise Exception( f"No record body found. BatchSize is probably not 1. event: {event}" ) body = json.loads(body) job = schemas.Job(**body) logger.info(event=QJLogEvents.InitJob, job=job) logger.info(event=QJLogEvents.RunQueryStart) query_result = run_query(job=job, config=query_config) logger.info(event=QJLogEvents.RunQueryEnd, num_results=query_result.get_length()) results: List[schemas.Result] = [] if query_config.account_id_key not in query_result.query_result_set.fields: raise Exception( f"Query results must contain field '{query_config.account_id_key}'" ) for q_r in query_result.to_list(): account_id = q_r[query_config.account_id_key] result = schemas.Result( account_id=account_id, result={ key: val for key, val in q_r.items() if key != query_config.account_id_key }, ) results.append(result) graph_spec = schemas.ResultSetGraphSpec( graph_uris_load_times=query_result.graph_uris_load_times) result_set = schemas.ResultSetCreate(job=job, graph_spec=graph_spec, results=results) api_key = get_api_key(region_name=query_config.region) qj_client = QJAPIClient(host=query_config.api_host, port=query_config.api_port, api_key=api_key) logger.info(event=QJLogEvents.CreateResultSetStart) qj_client.create_result_set(result_set=result_set) logger.info(event=QJLogEvents.CreateResultSetEnd)
def test_views_with_two_fresh_result_sets_one_missing_an_account_other_expired( self): """Add two result sets, one of which are within max_result_age_sec and the other is expired. The newer is missing one account. Run a query against the latest view to validate we get data for only one account (from the unexpired set)""" db_ro_user = "******" job_crud = CRUDJob( db_ro_user=db_ro_user, result_expiration_sec_default=int(1e6), result_expiration_sec_limit=int(1e6), max_graph_age_sec_default=int(1e6), max_graph_age_sec_limit=int(1e6), max_result_age_sec_default=int(1e6), max_result_age_sec_limit=int(1e6), account_id_key="test_account_id", ) result_set_crud = CRUDResultSet( max_result_set_results=int(1e6), max_result_size_bytes=int(1e6), job_crud=job_crud, ) with temp_db_session() as session: session.execute(f"CREATE ROLE {db_ro_user}") job_create = schemas.JobCreate( name="test_job", description="A Test Job", graph_spec=schemas.JobGraphSpec(graph_names=["test"]), category=schemas.Category.gov, severity=schemas.Severity.info, query= "select ?test_account_id ?foo ?boo where {?test_account_id ?foo ?boo} limit 10", max_graph_age_sec=int(1e6), result_expiration_sec=int(1e6), max_result_age_sec=int(1e6), notify_if_results=False, ) created_timestamp = job_crud.create( db_session=session, job_create_in=job_create).created # activate job_update = schemas.JobUpdate(active=True) _job = job_crud.update_version( db_session=session, job_name="test_job", created=created_timestamp, job_update=job_update, ) job = schemas.Job.from_orm(_job) account_id_a = "012345678901" account_id_b = "567890123456" result_set_1_time = datetime.now() - timedelta( seconds=job_create.max_result_age_sec + 1) result_set_1_graph_spec = schemas.ResultSetGraphSpec( graph_uris_load_times={"test": result_set_1_time.timestamp()}) results_1 = [ schemas.Result( account_id=account_id_a, result={ "foo": "oldhello_a", "boo": "oldthere_a" }, ), schemas.Result( account_id=account_id_a, result={ "foo": "oldboo_a", "boo": "oldfoo_a" }, ), schemas.Result( account_id=account_id_b, result={ "foo": "oldhello_b", "boo": "oldthere_b" }, ), schemas.Result( account_id=account_id_b, result={ "foo": "oldboo_b", "boo": "oldfoo_b" }, ), ] result_set_1_create = ResultSetCreate( job=job, graph_spec=result_set_1_graph_spec, results=results_1, created=result_set_1_time, ) result_set_crud.create(db_session=session, obj_in=result_set_1_create) result_set_2_time = datetime.now() result_set_2_graph_spec = schemas.ResultSetGraphSpec( graph_uris_load_times={"test": result_set_2_time.timestamp()}) results_2 = [ schemas.Result( account_id=account_id_a, result={ "foo": "newhello_a", "boo": "newthere_a" }, ), schemas.Result( account_id=account_id_a, result={ "foo": "newboo_a", "boo": "newfoo_a" }, ), ] result_set_2_create = ResultSetCreate( job=job, graph_spec=result_set_2_graph_spec, results=results_2, created=result_set_2_time, ) result_set_crud.create(db_session=session, obj_in=result_set_2_create) # check latest results latest_results = session.execute("select * from test_job_latest") latest_rows = latest_results.fetchall() self.assertSequenceEqual( sorted(latest_rows), sorted([ (result_set_2_time, account_id_a, "newhello_a", "newthere_a"), (result_set_2_time, account_id_a, "newboo_a", "newfoo_a"), ]), ) # check all results all_results = session.execute("select * from test_job_all") all_rows = all_results.fetchall() self.assertSequenceEqual( sorted(all_rows), sorted([ (result_set_1_time, account_id_a, "oldhello_a", "oldthere_a"), (result_set_1_time, account_id_a, "oldboo_a", "oldfoo_a"), (result_set_1_time, account_id_b, "oldhello_b", "oldthere_b"), (result_set_1_time, account_id_b, "oldboo_b", "oldfoo_b"), (result_set_2_time, account_id_a, "newhello_a", "newthere_a"), (result_set_2_time, account_id_a, "newboo_a", "newfoo_a"), ]), )
def test_views_with_expired_result_set(self): """Add a single result set which is older than max_result_age_sec. Validate the latest view returns no results, also validate all_view""" db_ro_user = "******" job_crud = CRUDJob( db_ro_user=db_ro_user, result_expiration_sec_default=int(1e6), result_expiration_sec_limit=int(1e6), max_graph_age_sec_default=int(1e6), max_graph_age_sec_limit=int(1e6), max_result_age_sec_default=int(1e6), max_result_age_sec_limit=int(1e6), account_id_key="test_account_id", ) result_set_crud = CRUDResultSet( max_result_set_results=int(1e6), max_result_size_bytes=int(1e6), job_crud=job_crud, ) with temp_db_session() as session: session.execute(f"CREATE ROLE {db_ro_user}") job_create = schemas.JobCreate( name="test_job", description="A Test Job", graph_spec=schemas.JobGraphSpec(graph_names=["test"]), category=schemas.Category.gov, severity=schemas.Severity.info, query= "select ?test_account_id ?foo ?boo where {?test_account_id ?foo ?boo} limit 10", max_graph_age_sec=int(1e6), result_expiration_sec=int(1e6), max_result_age_sec=int(1e6), notify_if_results=False, ) created_timestamp = job_crud.create( db_session=session, job_create_in=job_create).created # activate job_update = schemas.JobUpdate(active=True) _job = job_crud.update_version( db_session=session, job_name="test_job", created=created_timestamp, job_update=job_update, ) job = schemas.Job.from_orm(_job) account_id_a = "012345678901" account_id_b = "567890123456" result_set_1_time = datetime.now() - timedelta( seconds=job_create.max_result_age_sec + 1) result_set_1_graph_spec = schemas.ResultSetGraphSpec( graph_uris_load_times={"test": result_set_1_time.timestamp()}) results_1 = [ schemas.Result( account_id=account_id_a, result={ "foo": "oldhello_a", "boo": "oldthere_a" }, ), schemas.Result( account_id=account_id_a, result={ "foo": "oldboo_a", "boo": "oldfoo_a" }, ), schemas.Result( account_id=account_id_b, result={ "foo": "oldhello_b", "boo": "oldthere_b" }, ), schemas.Result( account_id=account_id_b, result={ "foo": "oldboo_b", "boo": "oldfoo_b" }, ), ] result_set_1_create = ResultSetCreate( job=job, graph_spec=result_set_1_graph_spec, results=results_1, created=result_set_1_time, ) result_set_crud.create(db_session=session, obj_in=result_set_1_create) # check latest results latest_results = session.execute("select * from test_job_latest") self.assertEqual(latest_results.rowcount, 0) # check all results all_results = session.execute("select * from test_job_all") all_rows = all_results.fetchall() self.assertSequenceEqual( sorted(all_rows), sorted([ (result_set_1_time, account_id_a, "oldhello_a", "oldthere_a"), (result_set_1_time, account_id_a, "oldboo_a", "oldfoo_a"), (result_set_1_time, account_id_b, "oldhello_b", "oldthere_b"), (result_set_1_time, account_id_b, "oldboo_b", "oldfoo_b"), ]), )