def test_whole_visualizer(self):
# prepare data
values = [0.2, 0.4, 0.3]
def colormap(value):
red = 0
green = 1
blue = value
alpha = 0.5
return (red, green, blue, alpha)
# create visualizer
vis = Visualizer(self.regions,
values,
colormap,
contours=[self.whole_region],
title="Test plot")
# call preparations
vis.normalize_values()
vis.render_colors()
vis.prepare()
# save to image
vis.save_image(self.filepath)
# check image
self.assertTrue(os.path.exists(self.filepath))
def test_render_colors(self):
# arrange
vis = Visualizer(self.regions, self.values, self.colormap)
# act
result = vis.render_colors()
# assert
self.assertIsNone(result)
self.assertListEqual(vis.colors, self.colors)
def _visualize(self):
# split db into units
dbs = self._split_db()
# process data
regions = []
values = []
for db in dbs:
# make region
geo = db[self.granularity].find_one({}, fields="geo")
region = Region.from_json(geo)
regions.append(region)
# evaluate value
value = self.function(db)
values.append(value)
# determine outline units
outline_geos = self.source_db[self.outlines_granularity].find(
{}, fields="geo")
outline_regions = [Region.from_json(geo) for geo in outline_geos]
# make visualizer object
self.vis = Visualizer(regions,
values,
self.colormap,
contours=outline_regions,
interpolation=self.interpolation,
title=self.title,
color_legend=self.show_legend,
grid=self.show_grid)
# normalize values if set
if self.normalization:
self.vis.normalize_values()
# apply colormap to values
self.vis.render_colors()
# prepare plot
self.vis.prepare()
### TODO # add title, legend, grid, values, etc.
# render plot to window or file
if self.output_filename:
visualized_dir = self.elections.visualized_dir
if not os.path.exists(visualized_dir):
### TODO - make image dir, not only main dir
os.makedirs(visualized_dir)
output_path = visualized_dir + self.output_filename
self.vis.save_image(output_path)
else:
self.vis.show()
def test_normalize_vector_values(self):
# arrange
vis = Visualizer(regions=self.regions,
values=[(0, 6, 7, 8, 9), (1, 2, 3, 4, 5)],
colormap=self.colormap,
normalization_range=[2, 5])
# act
result = vis.normalize_values()
# assert
self.assertIsNone(result)
self.assertListEqual(vis.values, [[2, 5, 5, 5, 5], [5, 2, 2, 2, 2]])
def test_normalize_values(self):
# arrange
vis = Visualizer(self.regions,
self.values,
self.colormap,
normalization_range=[0.5, 2])
# act
result = vis.normalize_values()
# assert
self.assertIsNone(result)
self.assertListEqual(vis.values, [2, 0.5])
def test_init_vector_values(self):
vis = Visualizer(regions=self.regions,
values=[(0, 6, 7, 8, 9), (1, 2, 3, 4, 5)],
colormap=self.colormap)
self.assertEqual(vis._vdim, 5)
self.assertEqual(len(vis.values), 2)
self.assertTupleEqual(vis.normalization_range,
((0, 1), (0, 1), (0, 1), (0, 1), (0, 1)))
vis_2 = Visualizer(regions=self.regions,
values=[(0, 6, 7, 8, 9), (1, 2, 3, 4, 5)],
colormap=self.colormap,
normalization_range=[(0, 1), (0, 1), (0, 1), (0, 1),
(0, 1)])
def test_show(self):
# arrange
mock_vis = MagicMock()
mock_plt = MagicMock()
# act
with patch("pkwscraper.lib.visualizer.plt", mock_plt):
result = Visualizer.show(mock_vis)
# assert
self.assertIsNone(result)
mock_plt.show.assert_called_once_with()
mock_plt.close.assert_called_once_with()
def test_save_image(self):
# arrange
mock_vis = MagicMock()
mock_plt = MagicMock()
filepath = "./directory/image001.svg"
# act
with patch("pkwscraper.lib.visualizer.plt", mock_plt):
result = Visualizer.save_image(mock_vis, filepath)
# assert
self.assertIsNone(result)
mock_plt.savefig.assert_called_once_with(filepath)
mock_plt.close.assert_called_once_with()
def test_init(self):
# act
with self.assertRaises(ValueError):
Visualizer(self.regions, [1, 2, 3, 4, 5], self.colormap)
with self.assertRaises(ValueError):
Visualizer(self.regions,
self.values,
self.colormap,
normalization_range=[1, 0])
vis = Visualizer(self.regions, self.values, self.colormap)
# assert
self.assertIsNone(vis._vdim)
self.assertListEqual(vis.regions, self.regions)
self.assertListEqual(vis.values, self.values)
self.assertIs(vis.colormap, self.colormap)
self.assertIsNone(vis.contours)
self.assertEqual(vis.interpolation, "linear")
self.assertTupleEqual(vis.normalization_range, (0, 1))
self.assertIsNone(vis.title)
self.assertFalse(vis.color_legend)
self.assertFalse(vis.grid)
def test_prepare(self):
# arrange
mock_2 = self.regions[1]
vis = Visualizer(self.regions,
self.values,
self.colormap,
contours=[mock_2])
vis.colors = self.colors
MockRegionClass = MagicMock()
mock_ax = MagicMock()
mock_fig = MagicMock()
mock_plt = MagicMock()
mock_plt.subplots.return_value = mock_fig, mock_ax
# act
with patch("pkwscraper.lib.visualizer.plt", mock_plt):
with patch("pkwscraper.lib.visualizer.Region", MockRegionClass):
result = vis.prepare()
# assert
mock_plt.subplots.assert_called_once_with()
mock_ax.set_xlim.assert_called_once_with(0, 4)
mock_ax.set_ylim.assert_called_once_with(1, 8)
self.assertEqual(mock_ax.add_collection.call_count, 2)
self.assertEqual(MockRegionClass.to_mpl_collection.call_count, 2)
def main():
# open DB
db = DbDriver(SEJM_2015_DATA_DIRECTORY, read_only=True)
# choose units to visualize
tables = ["gminy", "powiaty", "województwa", "okręgi"]
regions = []
for table_name in tables:
geos = db[table_name].find({}, fields="geo")
regions += [Region.from_json(geo) for geo in geos]
# prepare regions and values
n = len(regions)
values = n * [0]
colormap = lambda x: [random.random() for _ in range(3)] + [0.4]
# make visualizer
vis = Visualizer(regions, values, colormap)
vis.render_colors()
vis.prepare()
vis.show()
def test_with_colormap(self):
# prepare data
values = [(0.2, 0.5), (0.4, 0.1), (0.3, 1.0)]
colormap = Colormap(self.color_data_2d)
# create visualizer
vis = Visualizer(self.regions,
values,
colormap,
contours=[self.whole_region],
title="Test plot")
# call preparations
vis.normalize_values()
vis.render_colors()
class Controller:
"""
This is the main class of the project, that calls all steps
of data processing. This is created by passing main parameters
and, most importantly, evaluating function. This renders a plot
to image or for showing in separate window.
"""
def __init__(self,
elections,
function,
colormap,
granularity,
unit=None,
outlines_granularity=None,
normalization=True,
title=None,
show_legend=False,
show_grid=False,
output_filename=None,
interpolation='linear'):
"""
Constructor does basic checks and creates class attributes.
elections: (str, int) - type and year (unambiguous identifier)
of elections,
function: callable - function to evaluate data for single unit,
colormap: callable - function or object that converts numerical
values returned by function to proper colors,
granularity: str - the level of territorial units that plot
will be split into,
unit: (str, ID) or None - the unit to which analysis will be
limited; it is the pair of name of granularity, and then
the ID of specific unit (that means it has to be determined
earlier outside the class); if None - the plot is made for
the whole country,
outlines_granularity - level of territorial units that borders
will be placed on top of plot as contours,
normalization: bool - whether or not values from all units
should be scaled to (0,1) range before passing to colormap,
title: str - title of plot that is placed over the plot,
show_legend: bool - whether or not to show the color key in
form of legend, can contain extreme values written next
to it,
show_grid: bool - whether or not to show the frame around
the units plot,
output_filename: str or None - if None - the result will be
displayed in new window, otherwise, it will be rendered to
image file saved to given filenam in default visualizing
directory,
interpolation: str - method of interpolation of colors in the
colormap.
"""
# unpack unit
if unit is None:
unit_granularity = None
unit_id = None
else:
unit_granularity, unit_id = unit
# translate English variants of arguments
if granularity in GRANULARITY_DICT:
granularity = GRANULARITY_DICT[granularity]
if outlines_granularity in GRANULARITY_DICT:
outlines_granularity = GRANULARITY_DICT[outlines_granularity]
if unit_granularity in GRANULARITY_DICT:
unit_granularity = GRANULARITY_DICT[unit_granularity]
# basic correctness checks
if granularity not in GRANULARITY_DICT.values():
raise ValueError('`granularity` should be one of: "voivodships", '
'"constituencies", "districts" or "communes"')
if outlines_granularity not in GRANULARITY_DICT.values():
raise ValueError(
'`outlines_granularity` should be one of: "voivodships", '
'"constituencies", "districts" or "communes"')
if unit_granularity is not None \
and unit_granularity not in GRANULARITY_DICT.values():
raise ValueError(
'`unit` first part should be one of: "voivodships", '
'"constituencies", "districts" or "communes"')
if not isinstance(elections, tuple) or len(elections) != 2:
raise TypeError(
"Please, provide elections identifier: (type, year).")
# assing arguments
elections_type, year = elections
self.elections = Elections(elections_type=elections_type, year=year)
self.function = function
self.colormap = colormap
self.granularity = granularity
self.unit_granularity = unit_granularity
self.unit_id = unit_id
self.outlines_granularity = outlines_granularity
self.normalization = normalization
self.title = title
self.show_legend = show_legend
self.show_grid = show_grid
self.output_filename = output_filename
self.interpolation = interpolation
self.vis = None
self.source_db = None
def _scrape(self):
_ScraperClass = self.elections.get_scraper_class()
scraper = _ScraperClass()
scraper.run_all()
def _preprocess(self):
_PreprocessingClass = self.elections.get_preprocessing_class()
preprocessing = _PreprocessingClass()
preprocessing.run_all()
def _load_db(self):
try:
# try opening preprocessed db
DbDriver(self.elections.preprocessed_dir, read_only=True)
except IOError:
try:
# preprocessed db cannot be opened, check if there is rescribed db
DbDriver(self.elections.rescribed_dir, read_only=True)
except IOError:
# rescribed db cannot be opened, run downloading and scraping
self._scrape()
# rescribed db present, run preprocessing
self._preprocess()
# preprocessed db present, load it
self.source_db = DbDriver(self.elections.preprocessed_dir,
read_only=True)
def _split_db(self):
"""
This is used to split data in DB to correspond only to the
single unit of analysis. Function passed by user can use all
the DB instance data given to it, and be sure that they are
isolated from data corresponding to other units.
"""
# prepare indexes
db_refs = DbReferences(self.source_db, self.granularity)
# prepare units list
if self.unit_granularity is None:
units = self.source_db[self.granularity].find({})
else:
# check if unit is correctly set
self.source_db[self.unit_granularity][self.unit_id]
units = db_refs.get_relation(
_from=self.unit_granularity,
_to=self.granularity,
_id=self.unit_id,
)
# make DB driver instance for each unit
for unit_id in units:
# get IDs of records in tables
gmina_ids = db_refs.get_gmina(unit_id)
powiat_ids = db_refs.get_powiat(unit_id)
okreg_ids = db_refs.get_okreg(unit_id)
voivodship_ids = db_refs.get_voivodship(unit_id)
obwody_ids = db_refs.get_obwod(unit_id)
protocole_ids = db_refs.get_protocole(unit_id)
list_ids = db_refs.get_list(unit_id)
candidate_ids = db_refs.get_candidate(unit_id)
mandate_ids = db_refs.get_mandate(unit_id)
wyniki_ids = db_refs.get_wyniki(unit_id)
tables_and_ids = {
"gminy": gmina_ids,
"powiaty": powiat_ids,
"okręgi": okreg_ids,
"województwa": voivodship_ids,
"obwody": obwody_ids,
"protokoły": protocole_ids,
"listy": list_ids,
"kandydaci": candidate_ids,
"mandaty": mandate_ids
}
tables_and_ids.update(wyniki_ids)
# create db driver instance
db = DbDriver.__new__(DbDriver)
db._DbDriver__read_only = False
db._DbDriver__tables = {}
db._DbDriver__dropped_tables = []
# copy records
for table_name, ids_list in tables_and_ids.items():
db.create_table(table_name)
for _id in ids_list:
record = self.source_db[table_name][_id]
db[table_name].put(dict(record), _id=_id)
# freeze db and conclude iteration
db._DbDriver__read_only = True
yield db
def _visualize(self):
# split db into units
dbs = self._split_db()
# process data
regions = []
values = []
for db in dbs:
# make region
geo = db[self.granularity].find_one({}, fields="geo")
region = Region.from_json(geo)
regions.append(region)
# evaluate value
value = self.function(db)
values.append(value)
# determine outline units
outline_geos = self.source_db[self.outlines_granularity].find(
{}, fields="geo")
outline_regions = [Region.from_json(geo) for geo in outline_geos]
# make visualizer object
self.vis = Visualizer(regions,
values,
self.colormap,
contours=outline_regions,
interpolation=self.interpolation,
title=self.title,
color_legend=self.show_legend,
grid=self.show_grid)
# normalize values if set
if self.normalization:
self.vis.normalize_values()
# apply colormap to values
self.vis.render_colors()
# prepare plot
self.vis.prepare()
### TODO # add title, legend, grid, values, etc.
# render plot to window or file
if self.output_filename:
visualized_dir = self.elections.visualized_dir
if not os.path.exists(visualized_dir):
### TODO - make image dir, not only main dir
os.makedirs(visualized_dir)
output_path = visualized_dir + self.output_filename
self.vis.save_image(output_path)
else:
self.vis.show()
def run(self):
"""
Run prepared analysis object. It first makes sure the DB is
ready to use, or loads it and possibly runs preprocessing/etc.
"""
self._load_db()
self._visualize()
def show_db_schema(self):
""" Show tables and fields in DB as user guide. """
raise NotImplementedError("TODO")
return {tables: {columns: [values_type / enumerating]}}
pass