def stream_mapping(infile: Path,
outfile: Path,
mapping_yaml: Path,
sign: bool = True) -> None:
queries: List[Tuple[str, QueryMapping]] = []
config = load_mapping_file(mapping_yaml)
for dataset, meta in config.items():
for data in keys_values(meta, "queries", "query"):
data.pop("database", None)
data["csv_url"] = "/dev/null"
query = model.make_mapping(data, key_prefix=dataset)
queries.append((dataset, query))
try:
with path_writer(outfile) as outfh:
with input_file(infile) as fh:
for record in CSVSource.read_csv(fh):
for (dataset, query) in queries:
ns = Namespace(dataset)
if query.source.check_filters(record): # type: ignore
entities = query.map(record)
for entity in entities.values():
if sign:
entity = ns.apply(entity)
write_entity(outfh, entity)
except BrokenPipeError:
raise click.Abort()
def get_source(mapping):
"""Select the appropriate mapper to execute the given mapping."""
if 'database' in mapping.data:
return SQLSource(mapping, mapping.data)
elif 'csv_url' in mapping.data or 'csv_urls' in mapping.data:
return CSVSource(mapping, mapping.data)
raise InvalidMapping("Cannot determine mapping type")
def _get_source(self, data: Dict[str, Any]) -> Source:
if "database" in data:
return SQLSource(self, data)
if "csv_url" in data or "csv_urls" in data:
return CSVSource(self, data)
raise InvalidMapping("Cannot determine mapping type: %r" % data)
def source(self):
if "database" in self.data:
return SQLSource(self, self.data)
elif "csv_url" in self.data or "csv_urls" in self.data:
return CSVSource(self, self.data)
raise InvalidMapping("Cannot determine mapping type")
def source(self):
if 'database' in self.data:
return SQLSource(self, self.data)
elif 'csv_url' in self.data or 'csv_urls' in self.data:
return CSVSource(self, self.data)
raise InvalidMapping("Cannot determine mapping type")