def create_file_result_fixed(input_query: InputFixedQuery,
request: Request, # needed for json_logging.get_correlation_id to work correctly
authorization: str = Header(None),
processor: Processor = Depends(get_processor)):
"""
Create result set of data with temporality type fixed,
and write result to file. Returns name of file in response.
"""
log.info(
f'Entering /data/fixed/generate-file with input query: {input_query}'
)
user_id = authorize_user(authorization)
log.info(f"Authorized token for user: {user_id}")
result_data = processor.process_fixed_request(input_query)
resultset_file_name = processor.write_table(result_data)
log.info(f'File name for event result set: {resultset_file_name}')
return {
'filename': resultset_file_name,
}
def stream_result_fixed(input_query: InputFixedQuery,
request: Request, # needed for json_logging.get_correlation_id to work correctly
authorization: str = Header(None),
processor: Processor = Depends(get_processor)):
"""
Create result set of data with temporality type fixed,
and stream result as response.
"""
log.info(f'Entering /data/fixed/stream with input query: {input_query}')
user_id = authorize_user(authorization)
log.info(f"Authorized token for user: {user_id}")
result_data = processor.process_fixed_request(input_query)
buffer_stream = pa.BufferOutputStream()
pq.write_table(result_data, buffer_stream)
return PlainTextResponse(
buffer_stream.getvalue().to_pybytes()
)
import pytest
import os
import pyarrow.parquet as pq
from tests.resources import test_data
from data_service.config import config
from data_service.core.processor import (Processor)
from data_service.exceptions import NotFoundException
from data_service.core.filters import EmptyResultSetException
RESULTSET_DIR = 'tests/resources/resultset'
processor = Processor(settings=config.LocalFileSettings(
DATA_SERVICE_URL='https://fake-data-service-url',
DATASTORE_DIR='tests/resources/datastore_unit_test',
RESULTSET_DIR=RESULTSET_DIR))
def test_valid_event_request():
file_name = processor.process_event_request(
test_data.VALID_EVENT_QUERY_PERSON_INCOME_ALL)
assert parquet_table_to_csv_string(
file_name) == test_data.PERSON_INCOME_ALL
def test_valid_event_request_partitioned():
file_name = processor.process_event_request(
test_data.VALID_EVENT_QUERY_TEST_STUDIEPOENG_ALL)
assert parquet_table_to_csv_string(
file_name) == test_data.TEST_STUDIEPOENG_ALL
def get_processor():
return Processor(get_settings())