def test_graph_difference_more_both_isolated_non_isolated_relations_source(
stub_graph_set, stub_relation_set):
""" Tests graph_difference returns graph with expected nodes if source graph has non-isolated
and isolated nodes which are not present in target catalog """
_, vals = stub_graph_set
common_relation = Relation(name=rand_string(10),
**stub_relation_set.rand_relation_helper())
source_catalog = [
common_relation,
Relation(name=rand_string(10),
**stub_relation_set.rand_relation_helper()),
Relation(name=rand_string(10),
**stub_relation_set.rand_relation_helper()),
vals.downstream_relation, vals.upstream_relation, vals.birelation_right
]
target_catalog = {
common_relation,
Relation(name=rand_string(10),
**stub_relation_set.rand_relation_helper()),
Relation(name=rand_string(10),
**stub_relation_set.rand_relation_helper()),
vals.birelation_left, vals.birelation_right
}
config_dict = copy.deepcopy(BASIC_CONFIGURATION)
config_dict["source"]["specified_relations"] = [{
"database":
vals.downstream_relation.database,
"schema":
vals.downstream_relation.schema,
"relation":
vals.downstream_relation.name,
"relationships": {
"directional": [{
"local_attribute": vals.directional_key,
"database": ".*",
"schema": ".*",
"relation": ".*relation.*$",
"remote_attribute": vals.directional_key
}]
}
}]
config = ConfigurationParser().from_file_or_path(
StringIO(yaml.dump(config_dict)))
shgraph = SnowShuGraph()
with mock.MagicMock() as adapter_mock:
adapter_mock.build_catalog.return_value = source_catalog
config.source_profile.adapter = adapter_mock
shgraph.build_graph(config)
expected_nodes = source_catalog[1:]
actual = SnowShuGraph.catalog_difference(shgraph, target_catalog)
assert list(actual.nodes) == expected_nodes
def test_graph_difference_less_relations_source(stub_graph_set):
""" Tests graph_difference returns graph with no nodes if target catalog has more nodes
than source, including all nodes present in source graph """
shgraph = SnowShuGraph()
_, vals = stub_graph_set
source_catalog = [
vals.downstream_relation, vals.upstream_relation, vals.birelation_left,
vals.birelation_right
]
target_catalog = [
vals.downstream_relation, vals.upstream_relation, vals.birelation_left,
vals.birelation_right, vals.iso_relation
]
config_dict = copy.deepcopy(BASIC_CONFIGURATION)
config_dict["source"]["specified_relations"] = [{
"database":
vals.downstream_relation.database,
"schema":
vals.downstream_relation.schema,
"relation":
vals.downstream_relation.name,
"relationships": {
"directional": [{
"local_attribute": vals.directional_key,
"database": ".*",
"schema": ".*",
"relation": ".*relation.*$", # incl birelations
"remote_attribute": vals.directional_key
}]
}
}]
config = ConfigurationParser().from_file_or_path(
StringIO(yaml.dump(config_dict)))
with mock.MagicMock() as adapter_mock:
adapter_mock.build_catalog.return_value = source_catalog
config.source_profile.adapter = adapter_mock
shgraph.build_graph(config)
actual = SnowShuGraph.catalog_difference(shgraph, target_catalog)
assert len(actual) == 0
def test_graph_difference_empty_target(stub_graph_set):
""" Tests graph_difference returns source graph with all nodes if target catalog is empty """
shgraph = SnowShuGraph()
_, vals = stub_graph_set
full_catalog = [
vals.downstream_relation, vals.upstream_relation, vals.birelation_left,
vals.birelation_right
]
config_dict = copy.deepcopy(BASIC_CONFIGURATION)
config_dict["source"]["specified_relations"] = [{
"database":
vals.downstream_relation.database,
"schema":
vals.downstream_relation.schema,
"relation":
vals.downstream_relation.name,
"relationships": {
"directional": [{
"local_attribute": vals.directional_key,
"database": ".*",
"schema": ".*",
"relation": ".*relation.*$",
"remote_attribute": vals.directional_key
}]
}
}]
config = ConfigurationParser().from_file_or_path(
StringIO(yaml.dump(config_dict)))
with mock.MagicMock() as adapter_mock:
adapter_mock.build_catalog.return_value = full_catalog
config.source_profile.adapter = adapter_mock
shgraph.build_graph(config)
expected = shgraph.graph
target_catalog = set()
actual = SnowShuGraph.catalog_difference(shgraph, target_catalog)
assert actual == expected
def _execute(self,
barf: bool = False,
name: Optional[str] = None) -> Optional[str]:
graph = SnowShuGraph()
if name is not None:
self.config.name = name
graph.build_graph(self.config)
if self.incremental:
# TODO replica container should not be started for analyze commands
self.config.target_profile.adapter.initialize_replica(
self.config.source_profile.name, self.incremental)
incremental_target_catalog = self.config.target_profile.adapter.build_catalog(
patterns=SnowShuGraph.build_sum_patterns_from_configs(
self.config),
thread_workers=self.config.threads)
graph.graph = SnowShuGraph.catalog_difference(
graph.graph, incremental_target_catalog)
graphs = graph.get_connected_subgraphs()
if len(graphs) < 1:
args = (' new ', ' incremental ',
'; image up-to-date') if self.incremental else (' ', ' ',
'')
message = "No{}relations found per provided{}replica configuration{}, exiting.".format(
*args)
return message
if not self.config.target_profile.adapter.container:
# TODO replica container should not be started for analyze commands
self.config.target_profile.adapter.initialize_replica(
self.config.source_profile.name)
runner = GraphSetRunner()
runner.execute_graph_set(graphs,
self.config.source_profile.adapter,
self.config.target_profile.adapter,
threads=self.config.threads,
analyze=self.run_analyze,
barf=barf)
if not self.run_analyze:
relations = [
relation for graph in graphs for relation in graph.nodes
]
if self.config.source_profile.adapter.SUPPORTS_CROSS_DATABASE:
logger.info('Creating x-database links in target...')
self.config.target_profile.adapter.enable_cross_database()
logger.info('X-database enabled.')
self.config.target_profile.adapter.create_all_database_extensions()
logger.info('Applying %s emulation functions to target...',
self.config.source_profile.adapter.name)
for function in self.config.source_profile.adapter.SUPPORTED_FUNCTIONS:
self.config.target_profile.adapter.create_function_if_available(
function, relations)
logger.info('Emulation functions applied.')
self.config.target_profile.adapter.finalize_replica()
return printable_result(graph_to_result_list(graphs), self.run_analyze)
