def test_whole_visualizer(self): # prepare data values = [0.2, 0.4, 0.3] def colormap(value): red = 0 green = 1 blue = value alpha = 0.5 return (red, green, blue, alpha) # create visualizer vis = Visualizer(self.regions, values, colormap, contours=[self.whole_region], title="Test plot") # call preparations vis.normalize_values() vis.render_colors() vis.prepare() # save to image vis.save_image(self.filepath) # check image self.assertTrue(os.path.exists(self.filepath))
def test_render_colors(self): # arrange vis = Visualizer(self.regions, self.values, self.colormap) # act result = vis.render_colors() # assert self.assertIsNone(result) self.assertListEqual(vis.colors, self.colors)
def _visualize(self): # split db into units dbs = self._split_db() # process data regions = [] values = [] for db in dbs: # make region geo = db[self.granularity].find_one({}, fields="geo") region = Region.from_json(geo) regions.append(region) # evaluate value value = self.function(db) values.append(value) # determine outline units outline_geos = self.source_db[self.outlines_granularity].find( {}, fields="geo") outline_regions = [Region.from_json(geo) for geo in outline_geos] # make visualizer object self.vis = Visualizer(regions, values, self.colormap, contours=outline_regions, interpolation=self.interpolation, title=self.title, color_legend=self.show_legend, grid=self.show_grid) # normalize values if set if self.normalization: self.vis.normalize_values() # apply colormap to values self.vis.render_colors() # prepare plot self.vis.prepare() ### TODO # add title, legend, grid, values, etc. # render plot to window or file if self.output_filename: visualized_dir = self.elections.visualized_dir if not os.path.exists(visualized_dir): ### TODO - make image dir, not only main dir os.makedirs(visualized_dir) output_path = visualized_dir + self.output_filename self.vis.save_image(output_path) else: self.vis.show()
def test_normalize_vector_values(self): # arrange vis = Visualizer(regions=self.regions, values=[(0, 6, 7, 8, 9), (1, 2, 3, 4, 5)], colormap=self.colormap, normalization_range=[2, 5]) # act result = vis.normalize_values() # assert self.assertIsNone(result) self.assertListEqual(vis.values, [[2, 5, 5, 5, 5], [5, 2, 2, 2, 2]])
def test_normalize_values(self): # arrange vis = Visualizer(self.regions, self.values, self.colormap, normalization_range=[0.5, 2]) # act result = vis.normalize_values() # assert self.assertIsNone(result) self.assertListEqual(vis.values, [2, 0.5])
def test_init_vector_values(self): vis = Visualizer(regions=self.regions, values=[(0, 6, 7, 8, 9), (1, 2, 3, 4, 5)], colormap=self.colormap) self.assertEqual(vis._vdim, 5) self.assertEqual(len(vis.values), 2) self.assertTupleEqual(vis.normalization_range, ((0, 1), (0, 1), (0, 1), (0, 1), (0, 1))) vis_2 = Visualizer(regions=self.regions, values=[(0, 6, 7, 8, 9), (1, 2, 3, 4, 5)], colormap=self.colormap, normalization_range=[(0, 1), (0, 1), (0, 1), (0, 1), (0, 1)])
def test_show(self): # arrange mock_vis = MagicMock() mock_plt = MagicMock() # act with patch("pkwscraper.lib.visualizer.plt", mock_plt): result = Visualizer.show(mock_vis) # assert self.assertIsNone(result) mock_plt.show.assert_called_once_with() mock_plt.close.assert_called_once_with()
def test_save_image(self): # arrange mock_vis = MagicMock() mock_plt = MagicMock() filepath = "./directory/image001.svg" # act with patch("pkwscraper.lib.visualizer.plt", mock_plt): result = Visualizer.save_image(mock_vis, filepath) # assert self.assertIsNone(result) mock_plt.savefig.assert_called_once_with(filepath) mock_plt.close.assert_called_once_with()
def test_init(self): # act with self.assertRaises(ValueError): Visualizer(self.regions, [1, 2, 3, 4, 5], self.colormap) with self.assertRaises(ValueError): Visualizer(self.regions, self.values, self.colormap, normalization_range=[1, 0]) vis = Visualizer(self.regions, self.values, self.colormap) # assert self.assertIsNone(vis._vdim) self.assertListEqual(vis.regions, self.regions) self.assertListEqual(vis.values, self.values) self.assertIs(vis.colormap, self.colormap) self.assertIsNone(vis.contours) self.assertEqual(vis.interpolation, "linear") self.assertTupleEqual(vis.normalization_range, (0, 1)) self.assertIsNone(vis.title) self.assertFalse(vis.color_legend) self.assertFalse(vis.grid)
def test_prepare(self): # arrange mock_2 = self.regions[1] vis = Visualizer(self.regions, self.values, self.colormap, contours=[mock_2]) vis.colors = self.colors MockRegionClass = MagicMock() mock_ax = MagicMock() mock_fig = MagicMock() mock_plt = MagicMock() mock_plt.subplots.return_value = mock_fig, mock_ax # act with patch("pkwscraper.lib.visualizer.plt", mock_plt): with patch("pkwscraper.lib.visualizer.Region", MockRegionClass): result = vis.prepare() # assert mock_plt.subplots.assert_called_once_with() mock_ax.set_xlim.assert_called_once_with(0, 4) mock_ax.set_ylim.assert_called_once_with(1, 8) self.assertEqual(mock_ax.add_collection.call_count, 2) self.assertEqual(MockRegionClass.to_mpl_collection.call_count, 2)
def main(): # open DB db = DbDriver(SEJM_2015_DATA_DIRECTORY, read_only=True) # choose units to visualize tables = ["gminy", "powiaty", "województwa", "okręgi"] regions = [] for table_name in tables: geos = db[table_name].find({}, fields="geo") regions += [Region.from_json(geo) for geo in geos] # prepare regions and values n = len(regions) values = n * [0] colormap = lambda x: [random.random() for _ in range(3)] + [0.4] # make visualizer vis = Visualizer(regions, values, colormap) vis.render_colors() vis.prepare() vis.show()
def test_with_colormap(self): # prepare data values = [(0.2, 0.5), (0.4, 0.1), (0.3, 1.0)] colormap = Colormap(self.color_data_2d) # create visualizer vis = Visualizer(self.regions, values, colormap, contours=[self.whole_region], title="Test plot") # call preparations vis.normalize_values() vis.render_colors()
class Controller: """ This is the main class of the project, that calls all steps of data processing. This is created by passing main parameters and, most importantly, evaluating function. This renders a plot to image or for showing in separate window. """ def __init__(self, elections, function, colormap, granularity, unit=None, outlines_granularity=None, normalization=True, title=None, show_legend=False, show_grid=False, output_filename=None, interpolation='linear'): """ Constructor does basic checks and creates class attributes. elections: (str, int) - type and year (unambiguous identifier) of elections, function: callable - function to evaluate data for single unit, colormap: callable - function or object that converts numerical values returned by function to proper colors, granularity: str - the level of territorial units that plot will be split into, unit: (str, ID) or None - the unit to which analysis will be limited; it is the pair of name of granularity, and then the ID of specific unit (that means it has to be determined earlier outside the class); if None - the plot is made for the whole country, outlines_granularity - level of territorial units that borders will be placed on top of plot as contours, normalization: bool - whether or not values from all units should be scaled to (0,1) range before passing to colormap, title: str - title of plot that is placed over the plot, show_legend: bool - whether or not to show the color key in form of legend, can contain extreme values written next to it, show_grid: bool - whether or not to show the frame around the units plot, output_filename: str or None - if None - the result will be displayed in new window, otherwise, it will be rendered to image file saved to given filenam in default visualizing directory, interpolation: str - method of interpolation of colors in the colormap. """ # unpack unit if unit is None: unit_granularity = None unit_id = None else: unit_granularity, unit_id = unit # translate English variants of arguments if granularity in GRANULARITY_DICT: granularity = GRANULARITY_DICT[granularity] if outlines_granularity in GRANULARITY_DICT: outlines_granularity = GRANULARITY_DICT[outlines_granularity] if unit_granularity in GRANULARITY_DICT: unit_granularity = GRANULARITY_DICT[unit_granularity] # basic correctness checks if granularity not in GRANULARITY_DICT.values(): raise ValueError('`granularity` should be one of: "voivodships", ' '"constituencies", "districts" or "communes"') if outlines_granularity not in GRANULARITY_DICT.values(): raise ValueError( '`outlines_granularity` should be one of: "voivodships", ' '"constituencies", "districts" or "communes"') if unit_granularity is not None \ and unit_granularity not in GRANULARITY_DICT.values(): raise ValueError( '`unit` first part should be one of: "voivodships", ' '"constituencies", "districts" or "communes"') if not isinstance(elections, tuple) or len(elections) != 2: raise TypeError( "Please, provide elections identifier: (type, year).") # assing arguments elections_type, year = elections self.elections = Elections(elections_type=elections_type, year=year) self.function = function self.colormap = colormap self.granularity = granularity self.unit_granularity = unit_granularity self.unit_id = unit_id self.outlines_granularity = outlines_granularity self.normalization = normalization self.title = title self.show_legend = show_legend self.show_grid = show_grid self.output_filename = output_filename self.interpolation = interpolation self.vis = None self.source_db = None def _scrape(self): _ScraperClass = self.elections.get_scraper_class() scraper = _ScraperClass() scraper.run_all() def _preprocess(self): _PreprocessingClass = self.elections.get_preprocessing_class() preprocessing = _PreprocessingClass() preprocessing.run_all() def _load_db(self): try: # try opening preprocessed db DbDriver(self.elections.preprocessed_dir, read_only=True) except IOError: try: # preprocessed db cannot be opened, check if there is rescribed db DbDriver(self.elections.rescribed_dir, read_only=True) except IOError: # rescribed db cannot be opened, run downloading and scraping self._scrape() # rescribed db present, run preprocessing self._preprocess() # preprocessed db present, load it self.source_db = DbDriver(self.elections.preprocessed_dir, read_only=True) def _split_db(self): """ This is used to split data in DB to correspond only to the single unit of analysis. Function passed by user can use all the DB instance data given to it, and be sure that they are isolated from data corresponding to other units. """ # prepare indexes db_refs = DbReferences(self.source_db, self.granularity) # prepare units list if self.unit_granularity is None: units = self.source_db[self.granularity].find({}) else: # check if unit is correctly set self.source_db[self.unit_granularity][self.unit_id] units = db_refs.get_relation( _from=self.unit_granularity, _to=self.granularity, _id=self.unit_id, ) # make DB driver instance for each unit for unit_id in units: # get IDs of records in tables gmina_ids = db_refs.get_gmina(unit_id) powiat_ids = db_refs.get_powiat(unit_id) okreg_ids = db_refs.get_okreg(unit_id) voivodship_ids = db_refs.get_voivodship(unit_id) obwody_ids = db_refs.get_obwod(unit_id) protocole_ids = db_refs.get_protocole(unit_id) list_ids = db_refs.get_list(unit_id) candidate_ids = db_refs.get_candidate(unit_id) mandate_ids = db_refs.get_mandate(unit_id) wyniki_ids = db_refs.get_wyniki(unit_id) tables_and_ids = { "gminy": gmina_ids, "powiaty": powiat_ids, "okręgi": okreg_ids, "województwa": voivodship_ids, "obwody": obwody_ids, "protokoły": protocole_ids, "listy": list_ids, "kandydaci": candidate_ids, "mandaty": mandate_ids } tables_and_ids.update(wyniki_ids) # create db driver instance db = DbDriver.__new__(DbDriver) db._DbDriver__read_only = False db._DbDriver__tables = {} db._DbDriver__dropped_tables = [] # copy records for table_name, ids_list in tables_and_ids.items(): db.create_table(table_name) for _id in ids_list: record = self.source_db[table_name][_id] db[table_name].put(dict(record), _id=_id) # freeze db and conclude iteration db._DbDriver__read_only = True yield db def _visualize(self): # split db into units dbs = self._split_db() # process data regions = [] values = [] for db in dbs: # make region geo = db[self.granularity].find_one({}, fields="geo") region = Region.from_json(geo) regions.append(region) # evaluate value value = self.function(db) values.append(value) # determine outline units outline_geos = self.source_db[self.outlines_granularity].find( {}, fields="geo") outline_regions = [Region.from_json(geo) for geo in outline_geos] # make visualizer object self.vis = Visualizer(regions, values, self.colormap, contours=outline_regions, interpolation=self.interpolation, title=self.title, color_legend=self.show_legend, grid=self.show_grid) # normalize values if set if self.normalization: self.vis.normalize_values() # apply colormap to values self.vis.render_colors() # prepare plot self.vis.prepare() ### TODO # add title, legend, grid, values, etc. # render plot to window or file if self.output_filename: visualized_dir = self.elections.visualized_dir if not os.path.exists(visualized_dir): ### TODO - make image dir, not only main dir os.makedirs(visualized_dir) output_path = visualized_dir + self.output_filename self.vis.save_image(output_path) else: self.vis.show() def run(self): """ Run prepared analysis object. It first makes sure the DB is ready to use, or loads it and possibly runs preprocessing/etc. """ self._load_db() self._visualize() def show_db_schema(self): """ Show tables and fields in DB as user guide. """ raise NotImplementedError("TODO") return {tables: {columns: [values_type / enumerating]}} pass