def test_csv_output(self):
yaml = """
- object: foo
fields:
a: 1
c: 3
- object: foo
fields:
b: 2
d: 4
- object: bar
fields:
barb: 2
bard: 4
"""
with TemporaryDirectory() as t:
output_stream = CSVOutputStream(Path(t) / "csvoutput")
generate(StringIO(yaml), {}, output_stream)
messages = output_stream.close()
assert "foo.csv" in messages[0]
assert "bar.csv" in messages[1]
assert "csvw" in messages[2]
assert (Path(t) / "csvoutput" / "foo.csv").exists()
with open(Path(t) / "csvoutput" / "csvw_metadata.json") as f:
metadata = json.load(f)
assert {table["url"] for table in metadata["tables"]} == {
"foo.csv",
"bar.csv",
}
def do_output(self, yaml):
with TemporaryDirectory() as t:
output_stream = CSVOutputStream(Path(t) / "csvoutput")
results = generate(StringIO(yaml), {}, output_stream)
output_stream.close()
table_names = results.tables.keys()
tables = {}
for table in table_names:
pathname = Path(t) / "csvoutput" / (table + ".csv")
assert pathname.exists()
with open(pathname) as f:
tables[table] = list(csv.DictReader(f))
return tables
def configure_output_stream(dburls, mapping_file, output_format, output_files,
output_folder):
assert isinstance(output_files, (list, type(None)))
output_streams = [] # we allow multiple output streams
for dburl in dburls:
if mapping_file:
with click.open_file(mapping_file, "r") as f:
mappings = yaml.safe_load(f)
else:
mappings = None
output_streams.append(SqlOutputStream.from_url(dburl, mappings))
# JSON is the only output format (other than debug) that can go on stdout
if output_format == "json" and not output_files:
output_streams.append(JSONOutputStream(sys.stdout))
if output_format == "csv":
output_streams.append(CSVOutputStream(output_folder))
if output_files:
for path in output_files:
if output_folder:
path = Path(output_folder,
path) # put the file in the output folder
format = output_format or Path(path).suffix[1:]
if format == "json":
output_streams.append(JSONOutputStream(path))
elif format == "txt":
output_streams.append(DebugOutputStream(path))
elif format == "dot":
output_streams.append(GraphvizOutputStream(path))
elif format in graphic_file_extensions:
output_streams.append(ImageOutputStream(path, format))
else:
raise click.ClickException(
f"Unknown format or file extension: {format}")
if len(output_streams) == 0:
output_stream = DebugOutputStream()
elif len(output_streams) == 1:
output_stream = output_streams[0]
else:
output_stream = MultiplexOutputStream(output_streams)
try:
yield output_stream
finally:
for output_stream in output_streams:
try:
messages = output_stream.close()
except Exception as e:
messages = None
click.echo(f"Could not close {output_stream}: {str(e)}",
err=True)
if messages:
for message in messages:
click.echo(message)