def test_graph_difference_more_both_isolated_non_isolated_relations_source( stub_graph_set, stub_relation_set): """ Tests graph_difference returns graph with expected nodes if source graph has non-isolated and isolated nodes which are not present in target catalog """ _, vals = stub_graph_set common_relation = Relation(name=rand_string(10), **stub_relation_set.rand_relation_helper()) source_catalog = [ common_relation, Relation(name=rand_string(10), **stub_relation_set.rand_relation_helper()), Relation(name=rand_string(10), **stub_relation_set.rand_relation_helper()), vals.downstream_relation, vals.upstream_relation, vals.birelation_right ] target_catalog = { common_relation, Relation(name=rand_string(10), **stub_relation_set.rand_relation_helper()), Relation(name=rand_string(10), **stub_relation_set.rand_relation_helper()), vals.birelation_left, vals.birelation_right } config_dict = copy.deepcopy(BASIC_CONFIGURATION) config_dict["source"]["specified_relations"] = [{ "database": vals.downstream_relation.database, "schema": vals.downstream_relation.schema, "relation": vals.downstream_relation.name, "relationships": { "directional": [{ "local_attribute": vals.directional_key, "database": ".*", "schema": ".*", "relation": ".*relation.*$", "remote_attribute": vals.directional_key }] } }] config = ConfigurationParser().from_file_or_path( StringIO(yaml.dump(config_dict))) shgraph = SnowShuGraph() with mock.MagicMock() as adapter_mock: adapter_mock.build_catalog.return_value = source_catalog config.source_profile.adapter = adapter_mock shgraph.build_graph(config) expected_nodes = source_catalog[1:] actual = SnowShuGraph.catalog_difference(shgraph, target_catalog) assert list(actual.nodes) == expected_nodes
def test_graph_difference_less_relations_source(stub_graph_set): """ Tests graph_difference returns graph with no nodes if target catalog has more nodes than source, including all nodes present in source graph """ shgraph = SnowShuGraph() _, vals = stub_graph_set source_catalog = [ vals.downstream_relation, vals.upstream_relation, vals.birelation_left, vals.birelation_right ] target_catalog = [ vals.downstream_relation, vals.upstream_relation, vals.birelation_left, vals.birelation_right, vals.iso_relation ] config_dict = copy.deepcopy(BASIC_CONFIGURATION) config_dict["source"]["specified_relations"] = [{ "database": vals.downstream_relation.database, "schema": vals.downstream_relation.schema, "relation": vals.downstream_relation.name, "relationships": { "directional": [{ "local_attribute": vals.directional_key, "database": ".*", "schema": ".*", "relation": ".*relation.*$", # incl birelations "remote_attribute": vals.directional_key }] } }] config = ConfigurationParser().from_file_or_path( StringIO(yaml.dump(config_dict))) with mock.MagicMock() as adapter_mock: adapter_mock.build_catalog.return_value = source_catalog config.source_profile.adapter = adapter_mock shgraph.build_graph(config) actual = SnowShuGraph.catalog_difference(shgraph, target_catalog) assert len(actual) == 0
def test_graph_difference_empty_target(stub_graph_set): """ Tests graph_difference returns source graph with all nodes if target catalog is empty """ shgraph = SnowShuGraph() _, vals = stub_graph_set full_catalog = [ vals.downstream_relation, vals.upstream_relation, vals.birelation_left, vals.birelation_right ] config_dict = copy.deepcopy(BASIC_CONFIGURATION) config_dict["source"]["specified_relations"] = [{ "database": vals.downstream_relation.database, "schema": vals.downstream_relation.schema, "relation": vals.downstream_relation.name, "relationships": { "directional": [{ "local_attribute": vals.directional_key, "database": ".*", "schema": ".*", "relation": ".*relation.*$", "remote_attribute": vals.directional_key }] } }] config = ConfigurationParser().from_file_or_path( StringIO(yaml.dump(config_dict))) with mock.MagicMock() as adapter_mock: adapter_mock.build_catalog.return_value = full_catalog config.source_profile.adapter = adapter_mock shgraph.build_graph(config) expected = shgraph.graph target_catalog = set() actual = SnowShuGraph.catalog_difference(shgraph, target_catalog) assert actual == expected
def _execute(self, barf: bool = False, name: Optional[str] = None) -> Optional[str]: graph = SnowShuGraph() if name is not None: self.config.name = name graph.build_graph(self.config) if self.incremental: # TODO replica container should not be started for analyze commands self.config.target_profile.adapter.initialize_replica( self.config.source_profile.name, self.incremental) incremental_target_catalog = self.config.target_profile.adapter.build_catalog( patterns=SnowShuGraph.build_sum_patterns_from_configs( self.config), thread_workers=self.config.threads) graph.graph = SnowShuGraph.catalog_difference( graph.graph, incremental_target_catalog) graphs = graph.get_connected_subgraphs() if len(graphs) < 1: args = (' new ', ' incremental ', '; image up-to-date') if self.incremental else (' ', ' ', '') message = "No{}relations found per provided{}replica configuration{}, exiting.".format( *args) return message if not self.config.target_profile.adapter.container: # TODO replica container should not be started for analyze commands self.config.target_profile.adapter.initialize_replica( self.config.source_profile.name) runner = GraphSetRunner() runner.execute_graph_set(graphs, self.config.source_profile.adapter, self.config.target_profile.adapter, threads=self.config.threads, analyze=self.run_analyze, barf=barf) if not self.run_analyze: relations = [ relation for graph in graphs for relation in graph.nodes ] if self.config.source_profile.adapter.SUPPORTS_CROSS_DATABASE: logger.info('Creating x-database links in target...') self.config.target_profile.adapter.enable_cross_database() logger.info('X-database enabled.') self.config.target_profile.adapter.create_all_database_extensions() logger.info('Applying %s emulation functions to target...', self.config.source_profile.adapter.name) for function in self.config.source_profile.adapter.SUPPORTED_FUNCTIONS: self.config.target_profile.adapter.create_function_if_available( function, relations) logger.info('Emulation functions applied.') self.config.target_profile.adapter.finalize_replica() return printable_result(graph_to_result_list(graphs), self.run_analyze)