def energuide_zip_fixture(tmpdir: py._path.local.LocalPath,
energuide_fixture: str) -> str:
outfile = os.path.join(tmpdir, 'scrubbed_random_sample_xml.zip')
data = extractor.extract_data(energuide_fixture)
extractor.write_data(data, outfile)
return outfile
def test_write_data(tmpdir: py._path.local.LocalPath) -> None:
output_path = f'{tmpdir}/output.zip'
data = [
{
'foo': 1,
'BUILDER': '4K02E90020',
'EVAL_ID': '12149'
},
{
'bar': 2,
'baz': 3,
'BUILDER': '4K13D01404',
'EVAL_ID': '12148'
},
]
result = extractor.write_data(data, output_path)
assert result == (2, 0)
with zipfile.ZipFile(output_path, 'r') as output_file:
files = [
output_file.read('12149-4K02E90020'),
output_file.read('12148-4K13D01404')
]
assert [json.loads(file) for file in files] == data
def extract(infile: str, outfile: str, progress: bool) -> None:
LOGGER.info(f'Extracting data from {infile} into {outfile}')
if os.path.exists(outfile):
LOGGER.warning(f'Warning: file {outfile} exists. Overwriting.')
extracted = extractor.extract_data(infile, show_progress=progress)
records_written, records_failed = extractor.write_data(extracted, outfile)
LOGGER.info(
f'Finished extracting data into {outfile}. '
f'Successfully written: {records_written}. Failed: {records_failed}')
def test_write_bad_data(tmpdir: py._path.local.LocalPath) -> None:
output_path = f'{tmpdir}/output.zip'
data: typing.List[typing.Dict[str, typing.Any]] = [
{
'foo': 1,
'BUILDER': '4K02E90020',
'EVAL_ID': '12148'
},
{
'bar': 2,
'baz': 3
},
]
extractor.write_data(data, output_path)
with zipfile.ZipFile(output_path, 'r') as output:
assert len(output.namelist()) == 1
def energuide_zip_fixture(tmpdir: py._path.local.LocalPath, energuide_fixture: str) -> str:
outfile = f'{tmpdir}/randomized_energuide_data.zip'
data = extractor.extract_data(energuide_fixture)
extractor.write_data(data, outfile)
return outfile