def setUp(self):
self.parent_dir = os.path.dirname(
os.path.dirname(os.path.realpath(__file__)))
self.path_to_valid_cmf_des = os.path.join(
self.parent_dir, 'tests', 'test_data',
'fixture_cmf_description_flat_test.json')
self.cmf = CrashMoveFolder(self.path_to_valid_cmf_des)
def test_layer_props_and_cookbook_mismatch(self):
cmf = CrashMoveFolder(self.path_to_valid_cmf_des)
cmf.layer_properties = os.path.join(
self.parent_dir, 'tests', 'testfiles',
'fixture_layer_properties_four_layers.json')
cmf.map_definitions = os.path.join(
self.parent_dir, 'tests', 'testfiles',
'fixture_cookbook_1map_4layers.json')
test_lp = LayerProperties(cmf, "test", verify_on_creation=False)
# 1) map_cb and lp match
test_mcb = MapCookbook(cmf, test_lp, verify_on_creation=True)
self.assertIsInstance(test_mcb, MapCookbook)
# 2) layer in map_cb but not in lp
# 3) layer in lp but not in map_cb
# 4) unmatched layers in both lp and map_cb
testcases = (
('fixture_cookbook_1map_5layers.json',
("mainmap-tran-rds-ln-s0-allmaps")),
('fixture_cookbook_1map_3layers.json',
("locationmap_stle_stl_pt_s0_locationmaps")),
('fixture_cookbook_1map_mismatch_layers.json',
("mainmap-tran-rds-ln-s0-allmaps",
"locationmap_stle_stl_pt_s0_locationmaps")),
)
for cb_file, strings_in_ex_msg in testcases:
cmf.map_definitions = os.path.join(self.parent_dir, 'tests',
'testfiles', cb_file)
with self.assertRaises(ValueError) as ve:
MapCookbook(cmf, test_lp, verify_on_creation=True)
if six.PY2:
self.assertRegexpMatches(
str(ve.exception),
"One or more layer names occur in only one of these files")
for s in strings_in_ex_msg:
self.assertRegexpMatches(str(ve.exception), s)
else:
self.assertRegex(
str(ve.exception),
"One or more layer names occur in only one of these files")
for s in strings_in_ex_msg:
self.assertRegex(str(ve.exception), s)
def setUp(self):
self.parent_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
self.path_to_valid_cmf_des = os.path.join(self.parent_dir, 'example', 'cmf_description_flat_test.json')
self.path_to_invalid_cmf_des = os.path.join(
self.parent_dir, 'tests', 'testfiles', 'fixture_cmf_description_one_file_and_one_dir_not_valid.json')
self.cmf = CrashMoveFolder(self.path_to_valid_cmf_des, verify_on_creation=False)
self.lyr_props = LayerProperties(self.cmf, '', verify_on_creation=False)
def get_active_data_step_list(humev_config_path):
"""
Generates a list of Steps, each of which execucutes a suitable naming convention test
for a GIS dataset with the `2_Active_data` folder within the crash move folder.
"""
humev = Event(humev_config_path)
cmf = CrashMoveFolder(humev.cmf_descriptor_path)
nc = name_convention.NamingConvention(cmf.data_nc_definition)
return _step_builer(get_all_gisfiles(cmf), nc, 'data', cmf)
def setUp(self):
self.parent_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
self.cmf_descriptor_path = os.path.join(self.parent_dir, 'example', 'cmf_description_flat_test.json')
self.event_descriptor_path = os.path.join(self.parent_dir, 'example', 'event_description.json')
self.recipe_file = os.path.join(self.parent_dir, 'example', 'example_single_map_recipe.json')
self.non_existant_file = os.path.join(self.parent_dir, 'example', 'non-existant-file.json')
self.cmf = CrashMoveFolder(self.cmf_descriptor_path, verify_on_creation=False)
self.event = Event(self.event_descriptor_path)
# self.cmf = CrashMoveFolder(self.cmf_descriptor_path)
self.lyr_props = LayerProperties(self.cmf, '', verify_on_creation=False)
def setUp(self):
parent_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
self.dnc_json_path = os.path.join(
parent_dir, 'example', 'data_naming_convention.json')
self.test_files_dir = os.path.join(parent_dir, 'tests', 'testfiles', '.')
cmf_descriptor_path = os.path.join(
parent_dir, 'example', 'cmf_description.json')
self.cmf = CrashMoveFolder(cmf_descriptor_path, verify_on_creation=False)
self.cmf.dnc_lookup_dir = os.path.join(parent_dir, 'example')
def test_different_cmf_args(self):
# 1) test with valid cmf object
test_cmf = CrashMoveFolder(self.path_to_valid_cmf_des)
test_lp = LayerProperties(test_cmf, "test", verify_on_creation=False)
self.assertIsInstance(test_lp, LayerProperties)
# 2) test with valid cmf file
test_lp = LayerProperties(self.path_to_valid_cmf_des,
"test",
verify_on_creation=False)
self.assertIsInstance(test_lp, LayerProperties)
# 3) test with invalid cmf object (eg and CrashMoveFolder object
# where verify_paths() returns False)
test_cmf = CrashMoveFolder(self.path_to_invalid_cmf_des,
verify_on_creation=False)
self.assertRaises(ValueError, LayerProperties, test_cmf, "test")
# 4) test with invalid cmd file
self.assertRaises(ValueError, LayerProperties,
self.path_to_invalid_cmf_des, "test")
def check_lyr_props_vs_map_cookbook(self, **kwargs):
try:
cmf = CrashMoveFolder(self.cmf_desc_path)
lyrs = LayerProperties(cmf, '', verify_on_creation=False)
MapCookbook(cmf, lyrs, verify_on_creation=True)
return (
'No inconsistency detected between:\n'
' * the contents of the layer properties json file:\n\t{props}\n'
' * and the contents of the MapCookbook json:\n\t{cbook}\n'.
format(props=cmf.layer_properties, cbook=cmf.map_definitions))
except ValueError:
raise
def check_lyr_props_vs_rendering_dir(self, **kwargs):
cmf = CrashMoveFolder(self.cmf_desc_path)
for lyr_exn in self.lyr_file_exn_list:
try:
LayerProperties(cmf, lyr_exn, verify_on_creation=True)
return (
'No inconsistency detected between:\n'
' * the contents of the layer properties json file:\n\t{props}\n'
' * and layer rendering dir:\n\t{render}\n'.format(
props=cmf.layer_properties,
render=cmf.layer_rendering))
except ValueError:
raise
def setUp(self):
self.parent_dir = os.path.dirname(
os.path.dirname(os.path.realpath(__file__)))
self.dir_to_valid_cmf_des = os.path.join(self.parent_dir, 'example')
self.path_to_valid_cmf_des = os.path.join(
self.dir_to_valid_cmf_des, 'cmf_description_flat_test.json')
self.path_to_event_des = os.path.join(self.dir_to_valid_cmf_des,
'event_description.json')
self.cmf = CrashMoveFolder(self.path_to_valid_cmf_des)
self.lyr_props = LayerProperties(self.cmf,
'',
verify_on_creation=False)
self.dummy_runner = DummyRunner(Event(self.path_to_event_des))
def test_layer_props_and_cmf_mismatch(self):
# create and test_cmf and test_lp which refer different layer_prop.json files
cmf1 = CrashMoveFolder(self.path_to_valid_cmf_des)
test_lp = LayerProperties(cmf1, "test", verify_on_creation=False)
# now point the test_cmf to a different layer_props.json
cmf2 = CrashMoveFolder(self.path_to_valid_cmf_des)
cmf2.layer_properties = os.path.join(
self.parent_dir, 'tests', 'testfiles',
'fixture_layer_properties_four_layers.json')
# check that a ValueError is raised when `verify_on_creation=True`
with self.assertRaises(ValueError) as ve:
MapCookbook(cmf2, test_lp, verify_on_creation=True)
if six.PY2:
self.assertRegexpMatches(str(ve.exception), "strange results")
else:
self.assertRegex(str(ve.exception), "strange results")
# check that a ValueError is not raised when `verify_on_creation=False`
test_mcb = MapCookbook(cmf2, test_lp, verify_on_creation=False)
self.assertIsInstance(test_mcb, MapCookbook)
def setUp(self):
self.parent_dir = os.path.dirname(
os.path.dirname(os.path.realpath(__file__)))
self.path_to_valid_cmf_des = os.path.join(
self.parent_dir, 'tests', 'test_data',
'fixture_cmf_description_flat_test.json')
self.cmf = CrashMoveFolder(self.path_to_valid_cmf_des)
self.event = Event(
os.path.join(self.parent_dir, 'tests', 'test_data',
'event_description.json'))
self.my_mxd_fname = os.path.join(
self.parent_dir, 'tests', 'test_data',
'output_arcgis_10_6_reference_landscape_bottom.mxd')
self.layer_props = LayerProperties(self.cmf, '.lyr')
self.cookBook = MapCookbook(self.cmf, self.layer_props)
def test_load_recipe_with_layer_props_inc(self):
# Test that a MapRecipe read form json with only a layername, combined with the
# relevant LayerProperties is equal to a MapRecipe with all of the layerdetails embeded.
# layerpros with
cmf = CrashMoveFolder(
os.path.join(self.parent_dir, 'example',
'cmf_description_relative_paths_test.json'))
cmf.layer_properties = os.path.join(
self.parent_dir, 'tests', 'testfiles', 'cookbooks',
'fixture_layer_properties_for_atlas.json')
test_lp = LayerProperties(cmf, ".lyr", verify_on_creation=False)
# recipe with layer name only
recipe1 = MapRecipe(fixtures.recipe_with_layer_name_only, test_lp)
# combined recipe with layer props
recipe2 = MapRecipe(fixtures.recipe_with_layer_details_embedded,
test_lp)
self.assertEqual(recipe1, recipe2)
# 2) with non-matching layer props schema
recipe3 = MapRecipe(fixtures.recipe_with_positive_iso3_code, test_lp)
self.assertNotEqual(recipe1, recipe3)
def test_get_single_file_checker(self):
cmf = CrashMoveFolder(self.cmf_descriptor_path)
nc_desc_path = os.path.join(self.parent_dir, 'example',
'data_naming_convention.json')
nc = name_convention.NamingConvention(nc_desc_path)
passing_path = '/path/to/some/gisdata/206_bldg/ken_bldg_bdg_py_s4_osm_pp.shp'
func = check_naming_convention.get_single_file_checker(
passing_path, nc, cmf)
self.assertIn('parsable and valid', func().get_message)
failing_path = '/path/to/some/gisdata/202_admn/ken_admn_ad0_ln_s0_IEBC_pp_HDX.shp'
func = check_naming_convention.get_single_file_checker(
failing_path, nc, cmf)
self.assertRaises(ValueError, func)
def setUp(self):
self.parent_dir = os.path.dirname(
os.path.dirname(os.path.realpath(__file__)))
self.path_to_valid_cmf_des = os.path.join(
self.parent_dir, 'tests', 'test_data',
'fixture_cmf_description_flat_test.json')
self.cmf = CrashMoveFolder(self.path_to_valid_cmf_des)
self.event = Event(
os.path.join(self.cmf.path, 'event_description.json'))
self.arcmap_runner = arcmap_runner.ArcMapRunner(self.event)
# 1) insert map
self.df1 = mock.Mock(name='data_frame1')
self.df1.name = 'data_frame1'
self.df1.elementHeight = 19
self.df1.elementWidth = 17
# 2) main map
self.df2 = mock.Mock(name='data_frame2')
self.df2.name = 'data_frame2'
self.df2.elementHeight = 100
self.df2.elementWidth = 200
# 3) main map (same size different asspect ratio)
self.df3 = mock.Mock(name='data_frame3')
self.df3.name = 'data_frame3'
self.df3.elementHeight = 50
self.df3.elementWidth = 400
# 4) main map - identical to df3
self.df4 = mock.Mock(name='data_frame4')
self.df4.name = 'data_frame4'
self.df4.elementHeight = 50
self.df4.elementWidth = 400
# 5) main map - widest, but not the largest area
self.df5 = mock.Mock(name='data_frame5')
self.df5.name = 'data_frame5'
self.df5.elementHeight = 10
self.df5.elementWidth = 500
# 6) inset map - identical to df1 in every way including the name
self.df6 = mock.Mock(name='data_frame6')
self.df6.name = 'data_frame1'
self.df6.elementHeight = 19
self.df6.elementWidth = 17
def setUp(self):
self.parent_dir = os.path.dirname(
os.path.dirname(os.path.realpath(__file__)))
self.cmf_descriptor_path = os.path.join(
self.parent_dir, 'example', 'cmf_description_flat_test.json')
self.cmf = CrashMoveFolder(self.cmf_descriptor_path,
verify_on_creation=False)
self.event_descriptor_path = os.path.join(self.parent_dir, 'example',
'event_description.json')
self.event = Event(self.event_descriptor_path)
# self.cmf = CrashMoveFolder(self.cmf_descriptor_path)
self.lyr_props = LayerProperties(self.cmf,
'',
verify_on_creation=False)
self.recipe_descriptor_path = os.path.join(
self.parent_dir, 'example', 'example_single_map_recipe.json')
with open(self.recipe_descriptor_path) as json_file:
self.recipe_def = json.load(json_file)
self.recipe = MapRecipe(self.recipe_def, self.lyr_props)
def test_select_recipes(self):
# This looks for the MapID in the step.running_msg for the selected maps
# A list of tuples to test
# * The param (as entered by a human) to be passed to `get_cookbook_steps`. Includes upper, lower
# and mixed case, MapIDs that aren't in the cookbook etc. Can be either a string or a list
# * A list of the mapIDs that should be created (exactly as in the cookbook file - match case etc)
# * A list of mapID that should NOT be created (and represent a failure)
#
# This is the list of mapID that exist in the relevant cookbook
# ['MA001', 'MA002', 'MA003', 'MA004']
test_cases = [('ma001', ['MA001'], ['MA002', 'MA003', 'MA004']),
('MA004', ['MA004'], ['MA001', 'MA002', 'MA003']),
('Ma004', ['MA004'], ['MA001', 'MA002', 'MA003']),
('MA001', ['MA001'], ['MA002', 'MA003', 'MA004']),
(['mA001', 'ma002'], ['MA001',
'MA002'], ['MA003', 'MA004']),
('ma999', [], ['MA001', 'MA002', 'MA003', 'MA004']),
(None, ['MA001', 'MA002', 'MA003', 'MA004'], [])]
cmf = CrashMoveFolder(self.path_to_cmf_file)
lp = LayerProperties(cmf, 'test', False)
cb = MapCookbook(cmf, lp, False)
for mapid_arg, should_create, fail_list in test_cases:
# print('mapid_arg = {}'.format(mapid_arg))
# print('should_create = {}'.format(should_create))
# print('fail_list = {}'.format(fail_list))
should_create_set = set(should_create)
selected_recipes = plugin_controller.select_recipes(cb, mapid_arg)
selected_map_ids = set(
[recipe.mapnumber.upper() for recipe in selected_recipes])
# Check that no MapID were selected that shouldn't have been:
self.assertTrue(selected_map_ids.isdisjoint(fail_list))
# Check that all of the `selected_ids` match the `should_create` set:
self.assertEqual(should_create_set, selected_map_ids)
def __init__(self, cmf, extension, verify_on_creation=True):
"""
Positional Arguments:
* cmf: Either a CrashMoveFolder object or a path to a cmf_description.json file. If it
is a CrashMoveFolder object and cmf.verify_paths() returns False then an
ValueError exception will be rasied.
* extension: The file extension of the layer_rendering file type. (typically `.lyr` for
ESRI Layer files, `'.qml` for QGIS style)
Optional Named Arguments
verify_on_creation: True by default. If True then
`verify_match_with_layer_rendering_dir()` will be called from the constructor.
"""
try:
if cmf.verify_paths():
self.cmf = cmf
else:
raise ValueError(
'The `cmf` parameter for LayerProperties.__init__() can only accept'
'values where the paths verify. eg `cmf.verify_paths() == True`.'
'The value passed in this case failed this test')
except AttributeError:
self.cmf = CrashMoveFolder(cmf, verify_on_creation=True)
if len(extension) == 0 or extension.startswith('.'):
self.extension = extension
else:
self.extension = '.{}'.format(extension)
self.properties = {} # Dictionary
self._parse()
if verify_on_creation:
msg = self.get_difference_with_other_layer_set(
self._get_lyr_rendering_names_as_set(),
self._get_mismatch_with_layer_rendering_message)
if msg:
raise ValueError(msg)
def test_cmf_path_validation(self):
cmf_partial_fail = os.path.join(
self.parent_dir, 'tests', 'testfiles',
'fixture_cmf_description_one_file_and_one_dir_not_valid.json')
# test validation on creation (for failing case) using default parameters
self.assertRaises(ValueError, CrashMoveFolder, cmf_partial_fail)
# force validation on creation, for a json file we know would otherwise fail:
self.assertRaises(ValueError,
CrashMoveFolder,
cmf_partial_fail,
verify_on_creation=True)
# test validation is correctly disabled on creation, for a json file we know would otherwise fail:
test_cmf = CrashMoveFolder(cmf_partial_fail, verify_on_creation=False)
self.assertIsInstance(test_cmf, CrashMoveFolder)
# check message included in the ValueError:
with self.assertRaises(ValueError) as cm:
test_cmf = CrashMoveFolder(cmf_partial_fail,
verify_on_creation=True)
if six.PY2:
self.assertRegexpMatches(str(cm.exception), "map_templates")
self.assertNotRegexpMatches(str(cm.exception), "original_data")
else:
self.assertRegex(str(cm.exception), "map_templates")
self.assertNotRegex(str(cm.exception), "original_data")
# create a valid CMF object and then test paths, after creation
test_cmf_path = os.path.join(self.parent_dir, 'example',
'cmf_description_flat_test.json')
test_cmf = CrashMoveFolder(test_cmf_path)
self.assertTrue(test_cmf.verify_paths())
test_cmf_path = os.path.join(
self.parent_dir, 'example',
'cmf_description_relative_paths_test.json')
test_cmf = CrashMoveFolder(test_cmf_path)
self.assertTrue(test_cmf.verify_paths())
test_cmf.active_data = os.path.join(self.parent_dir, 'DOES-NOT-EXIST')
self.assertFalse(test_cmf.verify_paths())
def check_cmf_description(self, **kwargs):
self.cmf = CrashMoveFolder(self.cmf_desc_path)
return (
'The Crash Move Folder description file open correctly:\n"{}"\n'.
format(self.cmf_desc_path))
def test_verify_with_rendering_files(self):
# self.fail()
# load a valid CMF
test_cmf = CrashMoveFolder(self.path_to_valid_cmf_des)
# Overwright the Layer Properties file path with
# mapactionpy_controller\tests\testfiles\fixture_layer_properties_four_layers.json
test_cmf.layer_properties = os.path.join(
self.parent_dir, 'tests', 'testfiles',
'fixture_layer_properties_four_layers.json')
layer_rendering_test_root = os.path.join(self.parent_dir, 'tests',
'testfiles',
'test_layer_rendering')
test_cmf.layer_rendering
# 1) Exact match of .lyr files and layer properties
test_cmf.layer_rendering = os.path.join(layer_rendering_test_root,
'four_files_exact_match')
lyr_lp = LayerProperties(test_cmf, '.lyr', verify_on_creation=True)
self.assertFalse(lyr_lp.is_difference_with_layer_rendering_dir())
qml_lp = LayerProperties(test_cmf, '.qml', verify_on_creation=True)
self.assertFalse(qml_lp.is_difference_with_layer_rendering_dir())
# 2) .lyr files which don't have layer properties entries
test_cmf.layer_rendering = os.path.join(layer_rendering_test_root,
'five_files')
self.assertRaises(ValueError, LayerProperties, test_cmf, '.lyr')
self.assertRaises(ValueError, LayerProperties, test_cmf, '.qml')
# 3) layer properties entries which don't have cooresponding .lyr files.
test_cmf.layer_rendering = os.path.join(layer_rendering_test_root,
'three_files')
self.assertRaises(ValueError, LayerProperties, test_cmf, '.lyr')
self.assertRaises(ValueError, LayerProperties, test_cmf, '.qml')
# 4) Both 2 & 3 combined
test_cmf.layer_rendering = os.path.join(layer_rendering_test_root,
'four_files_mis_match')
self.assertRaises(ValueError, LayerProperties, test_cmf, '.lyr')
self.assertRaises(ValueError, LayerProperties, test_cmf, '.qml')
# 6) Overrided validation checks in constructor
test_cmf.layer_rendering = os.path.join(layer_rendering_test_root,
'four_files_mis_match')
long_lived_lp = LayerProperties(test_cmf,
'.lyr',
verify_on_creation=False)
self.assertTrue(long_lived_lp.is_difference_with_layer_rendering_dir())
long_lived_lp.cmf.layer_rendering = os.path.join(
layer_rendering_test_root, 'four_files_exact_match')
self.assertFalse(
long_lived_lp.is_difference_with_layer_rendering_dir())
# 7) Same results whether or not the '.' character is included in the file extension
test_cmf.layer_rendering = os.path.join(layer_rendering_test_root,
'four_files_exact_match')
# passing example with dot
lyr_lp = LayerProperties(test_cmf, '.lyr', verify_on_creation=True)
self.assertFalse(lyr_lp.is_difference_with_layer_rendering_dir())
# passing example without dot
lyr_lp = LayerProperties(test_cmf, 'lyr', verify_on_creation=True)
self.assertFalse(lyr_lp.is_difference_with_layer_rendering_dir())
# failing example with dot
test_cmf.layer_rendering = os.path.join(layer_rendering_test_root,
'four_files_mis_match')
self.assertRaises(ValueError, LayerProperties, test_cmf, '.lyr')
# failing example without dot
test_cmf.layer_rendering = os.path.join(layer_rendering_test_root,
'four_files_mis_match')
self.assertRaises(ValueError, LayerProperties, test_cmf, 'lyr')
def test_get_task_unique_summary(self):
dnc_json_path = os.path.join(self.parent_dir, 'example',
'data_naming_convention.json')
nc = NamingConvention(dnc_json_path)
cmf = CrashMoveFolder(self.path_to_valid_cmf_des)
# Three cases
# 1. Parsable and Valid
# 2. Parsable and Not Valid
# 3. Not Parsable
#
# key= is the name to test
# value= is a list of expected results from rendering the mustache elements in `mustache_tmpls`
test_cases = {
'lka_admn_ad2_py_s5_unocha_pp_freetext': [
'lka_admn_ad2_py_s5_unocha_pp_freetext',
'parsable but not valid',
('* "ad2" is a recognised value for the clause "datatheme" found in "03_theme.csv"\n'
'* "admn" is a recognised value for the clause "datacat" found in "02_category.csv"\n'
'* "freetext" is valid for freetext (as is almost anything)\n'
'* "lka" is a recognised value for the clause "geoext" found in "01_geoextent.csv"\n'
'* "pp" is a recognised value for the clause "perm" found in "07_permission.csv"\n'
'* "py" is a recognised value for the clause "geom" found in "04_geometry.csv"\n'
'* "s5" is a recognised value for the clause "scale" found in "05_scale.csv"\n'
'* "unocha" is a recognised value for the clause "source" found in "06_source.csv"\n'
), ''
],
'lka_admn_ad77_py_s5_unocha_pp_freetext': [
'lka_admn_ad77_py_s5_unocha_pp_freetext',
'parsable but not valid',
('* "admn" is a recognised value for the clause "datacat" found in "02_category.csv"\n'
'* "freetext" is valid for freetext (as is almost anything)\n'
'* "lka" is a recognised value for the clause "geoext" found in "01_geoextent.csv"\n'
'* "pp" is a recognised value for the clause "perm" found in "07_permission.csv"\n'
'* "py" is a recognised value for the clause "geom" found in "04_geometry.csv"\n'
'* "s5" is a recognised value for the clause "scale" found in "05_scale.csv"\n'
'* "unocha" is a recognised value for the clause "source" found in "06_source.csv"\n'
),
'* "ad77" is not a recognised value for the clause "datatheme" found in "03_theme.csv"\n'
],
'unparsable-name': ['unparsable-name', 'not parsable', '', '']
}
mustache_tmpls = [
'<%name_result.name_to_validate%>',
('<%#name_result.is_parsable%>parsable but not valid<%/name_result.is_parsable%>'
'<%^name_result.is_parsable%>not parsable<%/name_result.is_parsable%>'
),
'<%#name_result.valid_clause_list%>* <%&valid_clause%>\n<%/name_result.valid_clause_list%>',
'<%#name_result.invalid_clause_list %>* <%&invalid_clause%>\n<%/name_result.invalid_clause_list %>'
]
# for name_to_test, expected_result_list in test_cases.iteritems():
while test_cases:
name_to_test, expected_result_list = test_cases.popitem()
# print(name_to_test)
nr = nc.validate(name_to_test)
fdnt = FixDataNameTask(nr, cmf)
for test_template, expected_result in zip(mustache_tmpls,
expected_result_list):
# create the Task object
# override the unique identifier for test purposes
fdnt._primary_key_template = test_template
actual_result = fdnt.get_task_unique_summary()
self.assertEqual(actual_result, expected_result)
class BaseRunnerPlugin(object):
def __init__(self, hum_event, **kwargs):
self.hum_event = hum_event
self.cmf = CrashMoveFolder(self.hum_event.cmf_descriptor_path)
if not self.cmf.verify_paths():
raise ValueError(
"Cannot find paths and directories referenced by cmf {}".
format(self.cmf.path))
if self.__class__ is BaseRunnerPlugin:
raise NotImplementedError(
'BaseRunnerPlugin is an abstract class and cannot be instantiated directly'
)
def get_projectfile_extension(self, **kwargs):
raise NotImplementedError(
'BaseRunnerPlugin is an abstract class and the `get_projectfile_extension`'
' method cannot be called directly')
def get_lyr_render_extension(self, **kwargs):
raise NotImplementedError(
'BaseRunnerPlugin is an abstract class and the `get_lyr_render_extension`'
' method cannot be called directly')
def _get_all_templates_by_regex(self, recipe):
"""
Gets the fully qualified filenames of map templates, which exist in `self.cmf.map_templates` whose
filenames match the regex `recipe.template`.
@param recipe: A MapRecipe object.
@returns: A list of all of the templates, stored in `cmf.map_templates` whose
filename matches the regex `recipe.template` and that have the extention
`self.get_projectfile_extension()`
"""
def _is_relevant_file(f):
extension = os.path.splitext(f)[1]
logger.debug(
'checking file "{}", with extension "{}", against pattern "{}" and "{}"'
.format(f, extension, recipe.template,
self.get_projectfile_extension()))
if re.search(recipe.template, f):
logger.debug('file {} matched regex'.format(f))
f_path = os.path.join(self.cmf.map_templates, f)
logger.debug(
'file {} joined with self.cmf.map_templates "{}"'.format(
f, f_path))
return (os.path.isfile(f_path)) and (
extension == self.get_projectfile_extension())
else:
return False
# TODO: This results in calling `os.path.join` twice for certain files
logger.debug('searching for map templates in; {}'.format(
self.cmf.map_templates))
all_filenames = os.listdir(self.cmf.map_templates)
logger.debug('all available template files:\n\t{}'.format(
'\n\t'.join(all_filenames)))
relevant_filenames = [
os.path.realpath(os.path.join(self.cmf.map_templates, fi))
for fi in all_filenames if _is_relevant_file(fi)
]
logger.debug('possible template files:\n\t{}'.format(
'\n\t'.join(relevant_filenames)))
return relevant_filenames
def _get_template_by_aspect_ratio(self, template_aspect_ratios, target_ar):
"""
Selects the template which best matches the required aspect ratio.
@param possible_aspect_ratios: A list of tuples. For each tuple the first element is the path to the
template. The second element is the relevant aspect ratio for that
template. Typically this would be generated by
`get_aspect_ratios_of_templates()`.
@param target_ar: The taret aspect ratio - typically the aspect ratio of the bounding box for the country
being mapped.
@returns: The path of the template with the best matching aspect ratio.
"""
logger.info(
'Selecting from available templates based on the most best matching aspect ratio'
)
# Target is more landscape than the most landscape template
most_landscape = max(template_aspect_ratios, key=itemgetter(1))
if most_landscape[1] < target_ar:
logger.info(
'Target area of interest is more landscape than the most landscape template'
)
return most_landscape[0]
# Target is more portrait than the most portrait template
most_portrait = min(template_aspect_ratios, key=itemgetter(1))
if most_portrait[1] > target_ar:
logger.info(
'Target area of interest is more portrait than the most portrait template'
)
return most_portrait[0]
# The option with the smallest aspect ratio that is larger than target_ar
larger_ar = min([(templ_path, templ_ar)
for templ_path, templ_ar in template_aspect_ratios
if templ_ar >= target_ar],
key=itemgetter(1))
# The option with the largest aspect ratio that is smaller than target_ar
smaller_ar = max([(templ_path, templ_ar)
for templ_path, templ_ar in template_aspect_ratios
if templ_ar <= target_ar],
key=itemgetter(1))
# Linear combination:
# if (2*target_ar) > (larger_ar[1] + smaller_ar[1]):
# return larger_ar[0]
# asmith: personally I think that this is the better option, but will go with the linear combination for now
# logarithmic combination
if (2 * math.log(target_ar)) > (math.log(larger_ar[1]) +
math.log(smaller_ar[1])):
logger.info(
'Aspect ratio of the target area of interest lies between the aspect ratios of the'
' available templates')
return larger_ar[0]
return smaller_ar[0]
def get_aspect_ratios_of_templates(self, possible_templates, recipe):
"""
Plugins are required to implement this method.
The implementation should calculate the aspect ratio of the principal map frame within the list of
templates. The definition of "principal" is left to the plugin, though is typically the largest map
frame.
@param possible_templates: A list of paths to possible templates
@returns: A list of tuples. For each tuple the first element is the path to the template. The second
element is the aspect ratio of the largest* map frame within that template.
See `_get_largest_map_frame` for the description of how largest is determined.
@raises NotImplementedError: In the base class.
"""
raise NotImplementedError(
'BaseRunnerPlugin is an abstract class and the `get_aspect_ratios_of_templates`'
' method cannot be called directly')
def _get_aspect_ratio_of_bounds(self, bounds):
minx, miny, maxx, maxy = bounds
dx = (
maxx - minx
) % 360 # Accounts for the case where the bounds stradles the 180 meridian
dy = maxy - miny
return float(dx) / dy
def get_templates(self, **kwargs):
"""
Updates the recipe's `template_path` value. The result is the absolute path to the template.
To select the appropriate template it uses two inputs.
* The `recipe.template` value, which is a regex for the filename of the possible templates
* The target asspect ratio. If the aspect ratio of the target data can be determined then this is
also used to select the best matching template, amogst those which match the regex. If the
target ratio cannot be determined fromsource gis data, then the target ratio of 1.0 will be
used.
"""
recipe = kwargs['state']
# If there already is a valid `recipe.map_project_path` just skip with method
if recipe.map_project_path:
if os.path.exists(recipe.map_project_path):
return recipe
else:
raise ValueError(
'Unable to locate map project file: {}'.format(
recipe.map_project_path))
# use `recipe.template` as regex to locate one or more templates
possible_templates = self._get_all_templates_by_regex(recipe)
# Select the template with the most appropriate aspect ratio
possible_aspect_ratios = self.get_aspect_ratios_of_templates(
possible_templates, recipe)
mf = recipe.get_frame(recipe.principal_map_frame)
# Default value
target_aspect_ratio = 1.0
# If the MapFrame's target extent is not None, then use that:
if mf.extent:
target_aspect_ratio = self._get_aspect_ratio_of_bounds(mf.extent)
# use logic to workout which template has best aspect ratio
# obviously not this logic though:
recipe.template_path = self._get_template_by_aspect_ratio(
possible_aspect_ratios, target_aspect_ratio)
# TODO re-enable "Have the input files changed?"
# Have the input shapefiles changed?
return recipe
# TODO: asmith 2020/03/03
# 1) Please avoid hardcoding the naming convention for the mxds wherever possible. The Naming Convention
# classes can avoid the need to hardcode the naming convention for the input mxd templates. It might be
# possible to avoid the need to hardcode the naming convention for the output mxds using a
# String.Template be specified within the Cookbook?
# https://docs.python.org/2/library/string.html#formatspec
# https://www.python.org/dev/peps/pep-3101/
#
# 2) This only checks the filename for the mxd - it doesn't check the values within the text element of
# the map layout view (and hence the output metadata).
def get_next_map_version_number(self, mapNumberDirectory, mapNumber,
mapFileName):
versionNumber = 0
files = glob.glob(mapNumberDirectory + os.sep + mapNumber +
'-v[0-9][0-9]-' + mapFileName + '.mxd')
for file in files:
versionNumber = int(
os.path.basename(file).replace(mapNumber + '-v', '').replace(
('-' + mapFileName + '.mxd'), '')) # noqa
versionNumber = versionNumber + 1
return versionNumber
# TODO Is it possible to aviod the need to hardcode the naming convention for the output mxds? Eg could a
# String.Template be specified within the Cookbook?
# https://docs.python.org/2/library/string.html#formatspec
# https://www.python.org/dev/peps/pep-3101/
def create_ouput_map_project(self, **kwargs):
recipe = kwargs['state']
# Create `mapNumberDirectory` for output
output_dir = os.path.join(self.cmf.map_projects, recipe.mapnumber)
if not (os.path.isdir(output_dir)):
os.mkdir(output_dir)
# Construct output MXD/QPRJ name
logger.debug(
'About to create new map project file for product "{}"'.format(
recipe.product))
output_map_base = slugify(recipe.product)
logger.debug('Set output name for new map project file to "{}"'.format(
output_map_base))
recipe.version_num = self.get_next_map_version_number(
output_dir, recipe.mapnumber, output_map_base)
output_map_name = '{}-v{}-{}{}'.format(
recipe.mapnumber,
str(recipe.version_num).zfill(2), output_map_base,
self.get_projectfile_extension())
recipe.map_project_path = os.path.abspath(
os.path.join(output_dir, output_map_name))
logger.debug('Path for new map project file; {}'.format(
recipe.map_project_path))
logger.debug('Map Version number; {}'.format(recipe.version_num))
# Copy `src_template` to `recipe.map_project_path`
copyfile(recipe.template_path, recipe.map_project_path)
return recipe
def export_maps(self, **kwargs):
"""
Generates all file for export.
Accumulate some of the parameters for export XML, then calls
_do_export(....) to do that actual work
"""
recipe = kwargs['state']
export_params = {}
properties = {} # For properties from MapAction Toolbox
try:
export_params = self._create_export_dir(export_params, recipe)
if 'properties' in kwargs:
properties = kwargs['properties']
for key in list(properties.keys()):
export_params[str(key)] = properties[str(key)]
export_params = self._do_export(export_params, recipe)
except Exception as exp:
logger.error(
'Failed to export the map. export_params = "{}"'.format(
export_params))
logger.error(exp)
self.zip_exported_files(export_params)
def _create_export_dir(self, export_params, recipe):
# Accumulate parameters for export XML
version_str = "v" + str(recipe.version_num).zfill(2)
export_directory = os.path.abspath(
os.path.join(self.cmf.export_dir, recipe.mapnumber, version_str))
export_params["exportDirectory"] = export_directory
try:
os.makedirs(export_directory)
except OSError as exc: # Python >2.5
# Note 'errno.EEXIST' is not a typo. There should be two 'E's.
# https://docs.python.org/2/library/errno.html#errno.EEXIST
if exc.errno == errno.EEXIST and os.path.isdir(export_directory):
pass
else:
raise
return export_params
def _do_export(self, export_params, recipe):
raise NotImplementedError(
'BaseRunnerPlugin is an abstract class and the `export_maps`'
' method cannot be called directly')
def zip_exported_files(self, export_params):
# Get key params as local variables
core_file_name = export_params['coreFileName']
export_dir = export_params['exportDirectory']
mdr_xml_file_path = export_params['exportXmlFileLocation']
jpg_path = export_params['jpgFileLocation']
png_thumbnail_path = export_params['pngThumbNailFileLocation']
# And now Zip
zipFileName = core_file_name + ".zip"
zipFileLocation = os.path.join(export_dir, zipFileName)
with ZipFile(zipFileLocation, 'w') as zipObj:
zipObj.write(mdr_xml_file_path,
os.path.basename(mdr_xml_file_path))
zipObj.write(jpg_path, os.path.basename(jpg_path))
zipObj.write(png_thumbnail_path,
os.path.basename(png_thumbnail_path))
if (len(export_params.get("emfFileLocation", "")) > 0):
zipObj.write(
export_params['emfFileLocation'],
os.path.basename(export_params['emfFileLocation']))
# TODO: asmith 2020/03/03
# Given we are explictly setting the pdfFileName for each page within the DDPs
# it is possible return a list of all of the filenames for all of the PDFs. Please
# can we use this list to include in the zip file. There are edge cases where just
# adding all of the pdfs in a particular directory might not behave correctly (eg if
# the previous run had crashed midway for some reason)
for pdf in os.listdir(export_dir):
if pdf.endswith(".pdf"):
zipObj.write(
os.path.join(export_dir, pdf),
os.path.basename(os.path.join(export_dir, pdf)))
print("Export complete to " + export_dir)
def build_project_files(self, **kwargs):
raise NotImplementedError(
'BaseRunnerPlugin is an abstract class and the `build_project_files`'
' method cannot be called directly')
def test_cmf_description_adapter(self):
test_cmf = CrashMoveFolder(self.path_to_valid_cmf_des)
test_cd = task_renderer.cmf_description_adapter(test_cmf)
self.assertEqual(self.dir_to_valid_cmf_des, test_cd['cmf']['path'])
def test_layer_data_schema(self):
null_schema = True
passing_schema = yaml.safe_load(r"""
required:
- name_en
properties:
geometry_type:
items:
enum:
- MultiPolygon
- Polygon
additionalItems: false
crs:
items:
enum:
- EPSG:2090
additionalItems: false
""")
# Note the missing `enum` blocks
failing_schema = yaml.safe_load(r"""
required:
- name_en
properties:
geometry_type:
items:
- MultiPolygon
- Polygon
additionalItems: false
crs:
items:
- EPSG:2090
additionalItems: false
""")
cmf = CrashMoveFolder(
os.path.join(self.parent_dir, 'example', 'cmf_description_relative_paths_test.json'))
cmf.layer_properties = os.path.join(
self.parent_dir, 'tests', 'testfiles', 'cookbooks', 'fixture_layer_properties_for_atlas.json'
)
# Two cases where data schema is valid yaml
for test_schema in [null_schema, passing_schema]:
with mock.patch('mapactionpy_controller.data_schemas.yaml.safe_load') as mock_safe_load:
mock_safe_load.return_value = test_schema
test_lp = LayerProperties(cmf, ".lyr", verify_on_creation=False)
MapRecipe(fixtures.recipe_with_positive_iso3_code, test_lp)
self.assertTrue(True, 'validated jsonschema')
# case where data schema file itself malformed somehow
with mock.patch('mapactionpy_controller.data_schemas.yaml.safe_load') as mock_safe_load:
mock_safe_load.return_value = failing_schema
self.assertRaises(
jsonschema.exceptions.SchemaError,
MapRecipe,
fixtures.recipe_with_positive_iso3_code,
test_lp
)
def test_get_map_frame_extents(self):
cmf = CrashMoveFolder(
os.path.join(self.parent_dir, 'example',
'cmf_description_relative_paths_test.json'))
cmf.layer_properties = os.path.join(
self.parent_dir, 'tests', 'testfiles', 'cookbooks',
'fixture_layer_properties_for_atlas.json')
test_lp = LayerProperties(cmf, ".lyr", verify_on_creation=False)
# recipe with layer name only
with open(
os.path.join(
self.parent_dir, 'tests', 'testfiles',
'fixture_cookbook_1map_5layers_1frame.json')) as rf:
cookbook_def = json.load(rf)
# get the first (only) recipe in the cookbook
recipe_def = cookbook_def['recipes'].pop()
generic_lyr_def = json.loads('''{
"name": "the_name",
"reg_exp": "^wrl_admn_ad0_py_(.*?)_(.*?)_([phm][phm])(.+)shp$",
"schema_definition": "admin1_reference.yml",
"definition_query": "",
"display": true,
"add_to_legend": true,
"label_classes": []
}''')
# Case 1
# One or more layers does not have it's extent defined
case1_list = [('case1_lyrA', (35, 33, 36, 34), 'epsg:4326'),
('case1_lyrB', None, None)]
case1_result = (35, 33, 36, 34)
# Case 2
# Simple union with two lyrs of same crs
case2_list = [('case2_lyrA', (33, 51, 36, 58), 'epsg:4326'),
('case2_lyrB', (15, 52, 35, 55), 'epsg:4326')]
case2_result = (15, 51, 36, 58)
# Case 3
# Union with two lyrs of with different crs
# 'epsg:4326'== WGS1984, 'epsg:3785' == Web Mercator
case3_list = [('case3_lyrA', (33, 51, 36, 58), 'epsg:4326'),
('case3_lyrB', (1669792.36, 6800125.45, 3896182.18,
7361866.11), 'epsg:3785')]
case3_result = (15, 51, 36, 58)
# Case 4
# One layer which stradles 180 degree meridian
all_test_params = [(case1_list, case1_result),
(case2_list, case2_result),
(case3_list, case3_result)]
for test_list, expected_result in all_test_params:
test_recipe = MapRecipe(recipe_def, test_lp)
# Build up a mock list of layer to test
replacement_lyrs = []
for name, extent, crs in test_list:
new_lyr = RecipeLayer(generic_lyr_def,
test_lp,
verify_on_creation=False)
new_lyr.name = name
new_lyr.use_for_frame_extent = bool(extent)
# vaugely near Lebanon
new_lyr.extent = extent
new_lyr.crs = crs
replacement_lyrs.append(new_lyr)
# print('test_get_map_frame_extents')
# print('expected_result = {}'.format(expected_result))
test_frame = test_recipe.map_frames.pop()
test_frame.crs = 'epsg:4326'
test_frame.layers = replacement_lyrs
test_frame.calc_extent(state=test_recipe)
actual_result = test_frame.extent
# print('actual_result = {}'.format(actual_result))
for actual, expected in zip(actual_result, expected_result):
self.assertAlmostEqual(actual, expected)
def test_filter_lyr_for_use_in_frame_extent(self):
# Have included the digit at the start of the string, so that can be sorted easily.
cmf = CrashMoveFolder(
os.path.join(self.parent_dir, 'example',
'cmf_description_relative_paths_test.json'))
cmf.layer_properties = os.path.join(
self.parent_dir, 'tests', 'testfiles', 'cookbooks',
'fixture_layer_properties_for_atlas.json')
test_lp = LayerProperties(cmf, ".lyr", verify_on_creation=False)
# recipe with layer name only
with open(
os.path.join(
self.parent_dir, 'tests', 'testfiles',
'fixture_cookbook_1map_5layers_1frame.json')) as rf:
cookbook_def = json.load(rf)
# get the first (only) recipe in the cookbook
recipe_def = cookbook_def['recipes'].pop()
generic_lyr_def = json.loads('''{
"name": "the_name",
"reg_exp": "^wrl_admn_ad0_py_(.*?)_(.*?)_([phm][phm])(.+)shp$",
"schema_definition": "admin1_reference.yml",
"definition_query": "",
"display": true,
"add_to_legend": true,
"label_classes": []
}''')
# Case 1
# test white list
test_white_list1 = [('1a', True), ('1b', False), ('1c', None),
('1d', True), ('1e', False)]
expected_white_result1 = ['1a', '1d']
test_white_list2 = [('2a', True), ('2b', None), ('2c', None),
('2d', True), ('2e', None)]
expected_white_result2 = ['2a', '2d']
# Case 2
# Black List
test_black_list = [('3a', None), ('3b', False), ('3c', None),
('3d', None), ('3e', False)]
expected_black_result = ['3a', '3c', '3d']
# Case 3
# Default
test_default_list = [('4a', None), ('4b', None), ('4c', None),
('4d', None), ('4e', None)]
expected_default_result = ['4a', '4b', '4c', '4d', '4e']
all_test_params = [(test_white_list1, expected_white_result1),
(test_white_list2, expected_white_result2),
(test_black_list, expected_black_result),
(test_default_list, expected_default_result)]
for test_list, expected_result in all_test_params:
test_recipe = MapRecipe(recipe_def, test_lp)
# Build up a mock list of layer to test
replacement_lyrs = []
for test_lyr_details in test_list:
new_lyr = RecipeLayer(generic_lyr_def,
test_lp,
verify_on_creation=False)
new_lyr.name = test_lyr_details[0]
new_lyr.use_for_frame_extent = test_lyr_details[1]
# vaugely near Lebanon
new_lyr.extent = (35, 33, 36, 34)
new_lyr.crs = 'epsg:4326'
replacement_lyrs.append(new_lyr)
test_frame = test_recipe.map_frames.pop()
test_frame.layers = replacement_lyrs
result_lyrs = test_frame._filter_lyr_for_use_in_frame_extent()
actual_result = [lyr.name for lyr in result_lyrs]
self.assertEqual(actual_result, expected_result)