def both_schema_test(self, booking_file, hmis_file, url, expected_data):
with full_rig_with_s3() as (app, engine):
# Create matched jail_bookings
table_name = 'jail_bookings'
create_and_populate_master_table(table_name, engine, booking_file)
# Create matched hmis_service_stays
table_name = 'hmis_service_stays'
create_and_populate_master_table(table_name, engine, hmis_file)
response = app.get(url)
self.assertEqual(response.status_code, 200)
response_data = json.loads(response.get_data().decode('utf-8'))
set_of_venn_size = lambda venn: set(map(lambda x: x['size'], venn))
self.assertEqual(
set_of_venn_size(response_data['results']['vennDiagramData']),
set_of_venn_size(expected_data['results']['vennDiagramData']))
for expected_row, response_row in zip(
expected_data['results']['filteredData']['tableData'],
response_data['results']['filteredData']['tableData']):
self.assertDictEqual(expected_row, response_row)
for bar_key in [
'jailDurationBarData',
'homelessDurationBarData',
'jailContactBarData',
'homelessContactBarData',
]:
self.assertEqual(
expected_data['results']['filteredData'][bar_key],
response_data['results']['filteredData'][bar_key], bar_key)
def test_good_file(self, request_mock):
with full_rig_with_s3() as (app, engine):
upload_id = self.do_upload(app, request_mock)
# okay, here's what we really want to test.
# call the merge endpoint
response = app.post(
'/api/upload/merge_file?uploadId={}'.format(upload_id))
response_data = json.loads(response.get_data().decode('utf-8'))
assert response_data['status'] == 'success'
# make sure that there is a new merged file on s3
expected_s3_path = 's3://test-bucket/boone/hmis_service_stays/merged'
with open_sesame(expected_s3_path, 'rb') as expected_s3_file:
reader = csv.reader(expected_s3_file)
assert len([row for row in reader]) == ROWS_IN_GOOD_HMIS_FILE
# and make sure that the merge log has a record of this
assert db_session.query(MergeLog).filter(
MergeLog.upload_id == '123-456').one
# make sure that the master table has been bootstrapped
master_table = generate_master_table_name('boone',
'hmis_service_stays')
total_rows = db_session.query(
'count(*) from {}'.format(master_table)).one()
assert total_rows == (ROWS_IN_GOOD_HMIS_FILE - 1, )
# make sure that we filled in some matched ids
total_rows = db_session.query(
'count(matched_id is not null) from {}'.format(
master_table)).one()
assert total_rows == (ROWS_IN_GOOD_HMIS_FILE - 1, )
def test_error_transaction(self, request_mock):
with full_rig_with_s3() as (app, engine):
upload_id = self.do_upload(app, request_mock)
# try and merge an id that doesn't exist, should cause error
response = app.post(
'/api/upload/merge_file?uploadId={}'.format('garbage'))
response_data = json.loads(response.get_data().decode('utf-8'))
assert response_data['status'] == 'error'
# now merge the right one, the db should not be in a weird state
response = app.post(
'/api/upload/merge_file?uploadId={}'.format(upload_id))
response_data = json.loads(response.get_data().decode('utf-8'))
assert response_data['status'] == 'success'
def test_file_with_duplicates(self):
with full_rig_with_s3() as (app, engine):
response = app.post(
'/api/upload/upload_file?jurisdiction=boone&eventType=hmis_service_stays',
content_type='multipart/form-data',
data={
'file_field': (open(HMIS_FILE_WITH_DUPLICATES,
'rb'), 'myfile.csv')
})
response_data = json.loads(response.get_data().decode('utf-8'))
job_key = response_data['jobKey']
assert response_data['status'] == 'validating'
response = app.get('/api/upload/validated_result/' + job_key)
response_data = json.loads(response.get_data().decode('utf-8'))
assert 'validation' in response_data
assert response_data['validation']['status'] == 'invalid'
assert 'Duplicate key' in response_data['upload_result'][
'errorReport'][0]['message']
def test_good_file(self):
with full_rig_with_s3() as (app, engine):
response = app.post(
'/api/upload/upload_file?jurisdiction=boone&eventType=hmis_service_stays',
content_type='multipart/form-data',
data={
'file_field': (open(GOOD_HMIS_FILE, 'rb'), 'myfile.csv')
})
response_data = json.loads(response.get_data().decode('utf-8'))
assert response_data['status'] == 'validating'
assert 'jobKey' in response_data
assert 'message' in response_data
job_key = response_data['jobKey']
# get validation result and upload to s3
response = app.get('/api/upload/validated_result/' + job_key)
response_data = json.loads(response.get_data().decode('utf-8'))
assert 'validation' in response_data
assert response_data['validation']['status'] == 'valid'
assert response_data['validation']['jobKey'] == job_key
assert 'upload_result' in response_data
assert 'rowCount' in response_data['upload_result']
assert 'exampleRows' in response_data['upload_result']
assert 'uploadId' in response_data['upload_result']
assert 'fieldOrder' in response_data['upload_result']
current_date = date.today().isoformat()
expected_s3_path = 's3://test-bucket/boone/hmis_service_stays/uploaded/{}/{}'.format(
current_date, response_data['upload_result']['uploadId'])
with open_sesame(expected_s3_path) as expected_s3_file:
with open_sesame(GOOD_HMIS_FILE) as source_file:
# we do not expect the file on s3 to be the same as the
# uploaded source file - missing columns should be filled in
s3_df = pd.read_csv(expected_s3_file)
source_df = pd.read_csv(source_file, sep='|')
assert source_df.equals(s3_df[source_df.columns.tolist()])
assert db_session.query(Upload).filter(
Upload.id == response_data['upload_result']['uploadId']).one
def test_good_file(self, request_mock):
with full_rig_with_s3() as (app, engine):
upload_id = self.do_upload(app, request_mock)
# okay, here's what we really want to test.
# call the merge endpoint
response = app.post(
'/api/upload/merge_file?uploadId={}'.format(upload_id))
response_data = json.loads(response.get_data().decode('utf-8'))
assert response_data['status'] == 'success'
# make sure that there is a new merged file on s3
expected_s3_path = 's3://test-bucket/boone/jail_bookings/merged'
with open_sesame(expected_s3_path, 'rb') as expected_s3_file:
reader = csv.reader(expected_s3_file)
assert len([row for row in reader]) == 11
# and make sure that the merge log has a record of this
assert db_session.query(MergeLog).filter(
MergeLog.upload_id == '123-456').one
# and make sure that the raw table is no longer there
assert not table_exists(generate_raw_table_name(upload_id),
db_session.bind)
def test_download_file(self):
with full_rig_with_s3() as (app, engine):
# Create matched jail_bookings
table_name = 'jail_bookings'
create_and_populate_master_table(table_name, engine,
MATCHED_BOOKING_FILE)
# Create matched hmis_service_stays
table_name = 'hmis_service_stays'
create_and_populate_master_table(table_name, engine,
MATCHED_HMIS_FILE)
db_session.commit()
response = app.get(
'/api/chart/download_source?jurisdiction=boone&eventType=jail_bookings'
)
assert response.status_code == 200
assert response.headers[
"Content-Disposition"] == "attachment; filename=jail_bookings.csv"
assert response.headers["Content-type"] == "text/csv"
data = response.get_data()
reader = csv.reader(BytesIO(data))
with open(MATCHED_BOOKING_FILE, 'rb') as source_file:
source_reader = csv.reader(source_file)
for returned_row, expected_row in zip(reader, source_reader):
assert returned_row == expected_row