def assertForeignKeyConstraints(self, table, constraintsDescription):
"""Check that foreign-key constraints match the `constraintsDescription`.
"""
# Gather all actual constraints on the current table.
tableConstraints = {}
for constraint in table.foreign_key_constraints:
src = tuple(sorted(constraint.column_keys))
tgt = tuple(sorted((e.target_fullname for e in constraint.elements)))
tableConstraints[src] = tgt
# Check that all constraints in the description are indeed applied.
# Note that this is only a one-way check, the table may have more constraints imposed by other means.
for constraint in constraintsDescription:
src = tuple(sorted(iterable(constraint["src"])))
tgt = tuple(sorted(iterable(constraint["tgt"])))
self.assertIn(src, tableConstraints)
self.assertEqual(tableConstraints[src], tgt)
def keepOnly(config, toKeep):
"""Modify a SchemaConfig by dropping all tables not in the given list
"""
result = copy.deepcopy(config)
for name, description in config["tables"].items():
if name not in toKeep:
del result["tables", name]
else:
fks = []
for fk in description.get("foreignKeys", ()):
tgt = tuple(iterable(fk.get("tgt")))
table, _ = tgt[0].split(".")
if table in toKeep:
fks.append(fk)
result["tables", name, "foreignKeys"] = fks
return result
def _standardizeColumnParameter(
self, columns: Union[List[str], List[tuple],
Dict[str, Union[str, List[str]]]]
) -> Iterator[str]:
"""Transform a dictionary index into a multi-index column into a
string directly understandable by PyArrow.
Parameters
----------
columns : `dict`
Dictionary whose elements are string multi-index level names
and whose values are the value or values (as a list) for that
level.
Yields
------
name : `str`
Stringified tuple representing a multi-index column name.
"""
if isinstance(columns, list):
for requested in columns:
if not isinstance(requested, tuple):
raise ValueError(
"columns parameter for multi-index data frame"
"must be either a dictionary or list of tuples.")
yield str(requested)
else:
if not isinstance(columns, collections.abc.Mapping):
raise ValueError(
"columns parameter for multi-index data frame"
"must be either a dictionary or list of tuples.")
if not set(self.indexLevelNames).issuperset(columns.keys()):
raise ValueError(
f"Cannot use dict with keys {set(columns.keys())} "
f"to select columns from {self.indexLevelNames}.")
factors = [
iterable(columns.get(level, self.columns.levels[i]))
for i, level in enumerate(self.indexLevelNames)
]
for requested in itertools.product(*factors):
for i, value in enumerate(requested):
if value not in self.columns.levels[i]:
raise ValueError(
f"Unrecognized value {value!r}"
f"for index {self.indexLevelNames[i]!r}.")
yield str(requested)
def testIterableNoString(self):
self.assertEqual(list(iterable([0, 1, 2])), [0, 1, 2])
self.assertEqual(list(iterable(["hello", "world"])),
["hello", "world"])
def testString(self):
self.assertEqual(list(iterable("hello")), [
"hello",
])
def testNonIterable(self):
self.assertEqual(list(iterable(0)), [
0,
])
dataset_type_option = MWOptionDecorator(
"-d",
"--dataset-type",
callback=split_commas,
help="Specific DatasetType(s) to validate.",
multiple=True)
log_level_option = MWOptionDecorator(
"--log-level",
callback=partial(split_kv,
choice=click.Choice(
["CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG"],
case_sensitive=False),
normalize=True,
unseparated_okay=True),
default=iterable("WARNING"),
help="The Python log level to use.",
is_eager=True,
multiple=True)
long_log_option = MWOptionDecorator(
"--long-log",
help="Make log messages appear in long format.",
is_flag=True)
run_option = MWOptionDecorator(
"--output-run", help="The name of the run datasets should be output to.")
transfer_option = MWOptionDecorator(
"-t",
"--transfer",