def test_load_and_fetch():
path = os.path.join(os.path.dirname(__file__), 'test_table.csv')
bigquery.load_data_from_file(
'measures', 'test_table', path, TEST_SCHEMA)
result = list(bigquery.get_rows(
'ebmdatalab', 'measures', 'test_table'))
assert result[0] == {'id': 1, 'word': 'hello'}
assert result[1] == {'id': 2, 'word': 'goodbye'}
def create_bigquery_table():
"""Create a table in bigquery of all BNF codes for presentations that
are no longer current, along with the BNF code of their latest
incarnation
"""
dataset_name = 'hscic'
bq_table_name = 'bnf_map'
# output a row for each presentation and its ultimate replacement
with tempfile.NamedTemporaryFile(mode='r+b') as csv_file:
writer = csv.writer(csv_file)
for p in Presentation.objects.filter(replaced_by__isnull=False):
writer.writerow([p.bnf_code, p.current_version.bnf_code])
csv_file.seek(0)
load_data_from_file(dataset_name, bq_table_name, csv_file.name,
BNF_MAP_SCHEMA)
def test_load_data_from_file_with_exception(wait_mock, bigquery_mock):
with patch('tempfile.NamedTemporaryFile', create=True) \
as mock_tempfile:
writer_under_test = MagicMock(spec=file)
mock_tempfile.return_value = writer_under_test
with patch('ebmdatalab.bigquery.open', create=True) as mock_inputfile:
mock_csv = MagicMock(spec=file)
mock_csv.__enter__.return_value = iter(["1,foo", "2,bar"])
mock_inputfile.return_value = mock_csv
wait_mock.side_effect = RuntimeError(["foo"])
with patch('shutil.copyfile'):
with pytest.raises(RuntimeError) as e_info:
bigquery.load_data_from_file(
'dataset', 'table', 'source_mock',
[], _transform=lambda row: [row[0], row[1].upper()])
assert "Failed CSV has been copied" in e_info.exconly()
def test_load_data_from_file(wait_mock, bigquery_mock):
with patch('tempfile.NamedTemporaryFile', create=True) as mock_tempfile:
writer_under_test = MagicMock(spec=file)
mock_tempfile.return_value = writer_under_test
with patch('ebmdatalab.bigquery.open', create=True) as mock_inputfile:
mock_csv = MagicMock(spec=file)
mock_csv.__enter__.return_value = iter(["1,foo", "2,bar"])
mock_inputfile.return_value = mock_csv
bigquery.load_data_from_file(
'dataset', 'table', 'source_mock',
[], _transform=lambda row: [row[0], row[1].upper()])
# Check we read the passed-in filename
mock_inputfile.assert_called_once_with('source_mock', 'rb')
# Check the writer was called with transformed values
write_under_test = writer_under_test.__enter__.return_value.write
write_under_test.assert_any_call('1,FOO\r\n')
write_under_test.assert_any_call('2,BAR\r\n')
# Check bigquery `upload_from_file` was called
dataset = bigquery_mock.Client.return_value.dataset.return_value
assert dataset.table.return_value.upload_from_file.called
def load_test_data():
bigquery.load_data_from_file('test_hscic', 'prescribing_price_per_pill',
'tests/data.csv', bigquery.PRESCRIBING_SCHEMA)
def _createData(cls):
SHA.objects.create(code='Q51')
bassetlaw = PCT.objects.create(code='02Q', org_type='CCG')
lincs_west = PCT.objects.create(code='04D', org_type='CCG')
lincs_east = PCT.objects.create(code='03T',
org_type='CCG',
open_date='2013-04-01',
close_date='2015-01-01')
Chemical.objects.create(bnf_code='0703021Q0', chem_name='Desogestrel')
Chemical.objects.create(bnf_code='0408010A0',
chem_name='Levetiracetam')
Practice.objects.create(code='C84001',
ccg=bassetlaw,
name='LARWOOD SURGERY',
setting=4)
Practice.objects.create(code='C84024',
ccg=bassetlaw,
name='NEWGATE MEDICAL GROUP',
setting=4)
Practice.objects.create(code='B82005',
ccg=bassetlaw,
name='PRIORY MEDICAL GROUP',
setting=4,
open_date='2015-01-01')
Practice.objects.create(code='B82010',
ccg=bassetlaw,
name='RIPON SPA SURGERY',
setting=4)
Practice.objects.create(code='A85017',
ccg=bassetlaw,
name='BEWICK ROAD SURGERY',
setting=4)
Practice.objects.create(code='A86030',
ccg=bassetlaw,
name='BETTS AVENUE MEDICAL GROUP',
setting=4)
Practice.objects.create(code='C83051',
ccg=lincs_west,
name='ABBEY MEDICAL PRACTICE',
setting=4)
Practice.objects.create(code='C83019',
ccg=lincs_east,
name='BEACON MEDICAL PRACTICE',
setting=4)
# Ensure we only include open practices in our calculations.
Practice.objects.create(code='B82008',
ccg=bassetlaw,
name='NORTH SURGERY',
setting=4,
open_date='2010-04-01',
close_date='2012-01-01')
# Ensure we only include standard practices in our calculations.
Practice.objects.create(code='Y00581',
ccg=bassetlaw,
name='BASSETLAW DRUG & ALCOHOL SERVICE',
setting=1)
args = []
if 'SKIP_BQ_LOAD' not in os.environ:
fixtures_base = 'frontend/tests/fixtures/commands/'
prescribing_fixture = (fixtures_base +
'prescribing_bigquery_fixture.csv')
practices_fixture = fixtures_base + 'practices.csv'
bigquery.load_prescribing_data_from_file(
'measures', settings.BQ_PRESCRIBING_TABLE_NAME,
prescribing_fixture)
bigquery.load_data_from_file('measures',
settings.BQ_PRACTICES_TABLE_NAME,
practices_fixture,
bigquery.PRACTICE_SCHEMA)
month = '2015-09-01'
measure_id = 'cerazette'
args = []
opts = {
'month': month,
'measure': measure_id,
'test_mode': True,
'v': 3
}
call_command('import_measures', *args, **opts)