Example #1
0
 def setUp(self):
     self.parent_dir = os.path.dirname(
         os.path.dirname(os.path.realpath(__file__)))
     self.path_to_valid_cmf_des = os.path.join(
         self.parent_dir, 'tests', 'test_data',
         'fixture_cmf_description_flat_test.json')
     self.cmf = CrashMoveFolder(self.path_to_valid_cmf_des)
Example #2
0
    def test_layer_props_and_cookbook_mismatch(self):
        cmf = CrashMoveFolder(self.path_to_valid_cmf_des)
        cmf.layer_properties = os.path.join(
            self.parent_dir, 'tests', 'testfiles',
            'fixture_layer_properties_four_layers.json')
        cmf.map_definitions = os.path.join(
            self.parent_dir, 'tests', 'testfiles',
            'fixture_cookbook_1map_4layers.json')
        test_lp = LayerProperties(cmf, "test", verify_on_creation=False)

        # 1) map_cb and lp match
        test_mcb = MapCookbook(cmf, test_lp, verify_on_creation=True)
        self.assertIsInstance(test_mcb, MapCookbook)

        # 2) layer in map_cb but not in lp
        # 3) layer in lp but not in map_cb
        # 4) unmatched layers in both lp and map_cb
        testcases = (
            ('fixture_cookbook_1map_5layers.json',
             ("mainmap-tran-rds-ln-s0-allmaps")),
            ('fixture_cookbook_1map_3layers.json',
             ("locationmap_stle_stl_pt_s0_locationmaps")),
            ('fixture_cookbook_1map_mismatch_layers.json',
             ("mainmap-tran-rds-ln-s0-allmaps",
              "locationmap_stle_stl_pt_s0_locationmaps")),
        )

        for cb_file, strings_in_ex_msg in testcases:
            cmf.map_definitions = os.path.join(self.parent_dir, 'tests',
                                               'testfiles', cb_file)

            with self.assertRaises(ValueError) as ve:
                MapCookbook(cmf, test_lp, verify_on_creation=True)

            if six.PY2:
                self.assertRegexpMatches(
                    str(ve.exception),
                    "One or more layer names occur in only one of these files")
                for s in strings_in_ex_msg:
                    self.assertRegexpMatches(str(ve.exception), s)
            else:
                self.assertRegex(
                    str(ve.exception),
                    "One or more layer names occur in only one of these files")
                for s in strings_in_ex_msg:
                    self.assertRegex(str(ve.exception), s)
Example #3
0
    def setUp(self):
        self.parent_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
        self.path_to_valid_cmf_des = os.path.join(self.parent_dir, 'example', 'cmf_description_flat_test.json')
        self.path_to_invalid_cmf_des = os.path.join(
            self.parent_dir, 'tests', 'testfiles', 'fixture_cmf_description_one_file_and_one_dir_not_valid.json')

        self.cmf = CrashMoveFolder(self.path_to_valid_cmf_des, verify_on_creation=False)
        self.lyr_props = LayerProperties(self.cmf, '', verify_on_creation=False)
def get_active_data_step_list(humev_config_path):
    """
    Generates a list of Steps, each of which execucutes a suitable naming convention test
    for a GIS dataset with the `2_Active_data` folder within the crash move folder.
    """
    humev = Event(humev_config_path)
    cmf = CrashMoveFolder(humev.cmf_descriptor_path)
    nc = name_convention.NamingConvention(cmf.data_nc_definition)
    return _step_builer(get_all_gisfiles(cmf), nc, 'data', cmf)
 def setUp(self):
     self.parent_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
     self.cmf_descriptor_path = os.path.join(self.parent_dir, 'example', 'cmf_description_flat_test.json')
     self.event_descriptor_path = os.path.join(self.parent_dir, 'example', 'event_description.json')
     self.recipe_file = os.path.join(self.parent_dir, 'example', 'example_single_map_recipe.json')
     self.non_existant_file = os.path.join(self.parent_dir, 'example', 'non-existant-file.json')
     self.cmf = CrashMoveFolder(self.cmf_descriptor_path, verify_on_creation=False)
     self.event = Event(self.event_descriptor_path)
     # self.cmf = CrashMoveFolder(self.cmf_descriptor_path)
     self.lyr_props = LayerProperties(self.cmf, '', verify_on_creation=False)
    def setUp(self):
        parent_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))

        self.dnc_json_path = os.path.join(
            parent_dir, 'example', 'data_naming_convention.json')

        self.test_files_dir = os.path.join(parent_dir, 'tests', 'testfiles', '.')

        cmf_descriptor_path = os.path.join(
            parent_dir, 'example', 'cmf_description.json')
        self.cmf = CrashMoveFolder(cmf_descriptor_path, verify_on_creation=False)
        self.cmf.dnc_lookup_dir = os.path.join(parent_dir, 'example')
    def test_different_cmf_args(self):
        # 1) test with valid cmf object
        test_cmf = CrashMoveFolder(self.path_to_valid_cmf_des)
        test_lp = LayerProperties(test_cmf, "test", verify_on_creation=False)
        self.assertIsInstance(test_lp, LayerProperties)

        # 2) test with valid cmf file
        test_lp = LayerProperties(self.path_to_valid_cmf_des,
                                  "test",
                                  verify_on_creation=False)
        self.assertIsInstance(test_lp, LayerProperties)

        # 3) test with invalid cmf object (eg and CrashMoveFolder object
        #    where verify_paths() returns False)
        test_cmf = CrashMoveFolder(self.path_to_invalid_cmf_des,
                                   verify_on_creation=False)
        self.assertRaises(ValueError, LayerProperties, test_cmf, "test")

        # 4) test with invalid cmd file
        self.assertRaises(ValueError, LayerProperties,
                          self.path_to_invalid_cmf_des, "test")
 def check_lyr_props_vs_map_cookbook(self, **kwargs):
     try:
         cmf = CrashMoveFolder(self.cmf_desc_path)
         lyrs = LayerProperties(cmf, '', verify_on_creation=False)
         MapCookbook(cmf, lyrs, verify_on_creation=True)
         return (
             'No inconsistency detected between:\n'
             ' * the contents of the layer properties json file:\n\t{props}\n'
             ' * and the contents of the MapCookbook json:\n\t{cbook}\n'.
             format(props=cmf.layer_properties, cbook=cmf.map_definitions))
     except ValueError:
         raise
 def check_lyr_props_vs_rendering_dir(self, **kwargs):
     cmf = CrashMoveFolder(self.cmf_desc_path)
     for lyr_exn in self.lyr_file_exn_list:
         try:
             LayerProperties(cmf, lyr_exn, verify_on_creation=True)
             return (
                 'No inconsistency detected between:\n'
                 ' * the contents of the layer properties json file:\n\t{props}\n'
                 ' * and layer rendering dir:\n\t{render}\n'.format(
                     props=cmf.layer_properties,
                     render=cmf.layer_rendering))
         except ValueError:
             raise
 def setUp(self):
     self.parent_dir = os.path.dirname(
         os.path.dirname(os.path.realpath(__file__)))
     self.dir_to_valid_cmf_des = os.path.join(self.parent_dir, 'example')
     self.path_to_valid_cmf_des = os.path.join(
         self.dir_to_valid_cmf_des, 'cmf_description_flat_test.json')
     self.path_to_event_des = os.path.join(self.dir_to_valid_cmf_des,
                                           'event_description.json')
     self.cmf = CrashMoveFolder(self.path_to_valid_cmf_des)
     self.lyr_props = LayerProperties(self.cmf,
                                      '',
                                      verify_on_creation=False)
     self.dummy_runner = DummyRunner(Event(self.path_to_event_des))
Example #11
0
    def test_layer_props_and_cmf_mismatch(self):

        # create and test_cmf and test_lp which refer different layer_prop.json files
        cmf1 = CrashMoveFolder(self.path_to_valid_cmf_des)
        test_lp = LayerProperties(cmf1, "test", verify_on_creation=False)
        # now point the test_cmf to a different layer_props.json
        cmf2 = CrashMoveFolder(self.path_to_valid_cmf_des)
        cmf2.layer_properties = os.path.join(
            self.parent_dir, 'tests', 'testfiles',
            'fixture_layer_properties_four_layers.json')

        # check that a ValueError is raised when `verify_on_creation=True`
        with self.assertRaises(ValueError) as ve:
            MapCookbook(cmf2, test_lp, verify_on_creation=True)

        if six.PY2:
            self.assertRegexpMatches(str(ve.exception), "strange results")
        else:
            self.assertRegex(str(ve.exception), "strange results")

        # check that a ValueError is not raised when `verify_on_creation=False`
        test_mcb = MapCookbook(cmf2, test_lp, verify_on_creation=False)
        self.assertIsInstance(test_mcb, MapCookbook)
 def setUp(self):
     self.parent_dir = os.path.dirname(
         os.path.dirname(os.path.realpath(__file__)))
     self.path_to_valid_cmf_des = os.path.join(
         self.parent_dir, 'tests', 'test_data',
         'fixture_cmf_description_flat_test.json')
     self.cmf = CrashMoveFolder(self.path_to_valid_cmf_des)
     self.event = Event(
         os.path.join(self.parent_dir, 'tests', 'test_data',
                      'event_description.json'))
     self.my_mxd_fname = os.path.join(
         self.parent_dir, 'tests', 'test_data',
         'output_arcgis_10_6_reference_landscape_bottom.mxd')
     self.layer_props = LayerProperties(self.cmf, '.lyr')
     self.cookBook = MapCookbook(self.cmf, self.layer_props)
Example #13
0
    def test_load_recipe_with_layer_props_inc(self):
        # Test that a MapRecipe read form json with only a layername, combined with the
        # relevant LayerProperties is equal to a MapRecipe with all of the layerdetails embeded.

        # layerpros with
        cmf = CrashMoveFolder(
            os.path.join(self.parent_dir, 'example',
                         'cmf_description_relative_paths_test.json'))
        cmf.layer_properties = os.path.join(
            self.parent_dir, 'tests', 'testfiles', 'cookbooks',
            'fixture_layer_properties_for_atlas.json')
        test_lp = LayerProperties(cmf, ".lyr", verify_on_creation=False)

        # recipe with layer name only
        recipe1 = MapRecipe(fixtures.recipe_with_layer_name_only, test_lp)
        # combined recipe with layer props
        recipe2 = MapRecipe(fixtures.recipe_with_layer_details_embedded,
                            test_lp)

        self.assertEqual(recipe1, recipe2)

        # 2) with non-matching layer props schema
        recipe3 = MapRecipe(fixtures.recipe_with_positive_iso3_code, test_lp)
        self.assertNotEqual(recipe1, recipe3)
Example #14
0
    def test_get_single_file_checker(self):
        cmf = CrashMoveFolder(self.cmf_descriptor_path)
        nc_desc_path = os.path.join(self.parent_dir, 'example',
                                    'data_naming_convention.json')
        nc = name_convention.NamingConvention(nc_desc_path)

        passing_path = '/path/to/some/gisdata/206_bldg/ken_bldg_bdg_py_s4_osm_pp.shp'
        func = check_naming_convention.get_single_file_checker(
            passing_path, nc, cmf)
        self.assertIn('parsable and valid', func().get_message)

        failing_path = '/path/to/some/gisdata/202_admn/ken_admn_ad0_ln_s0_IEBC_pp_HDX.shp'
        func = check_naming_convention.get_single_file_checker(
            failing_path, nc, cmf)
        self.assertRaises(ValueError, func)
Example #15
0
    def setUp(self):
        self.parent_dir = os.path.dirname(
            os.path.dirname(os.path.realpath(__file__)))
        self.path_to_valid_cmf_des = os.path.join(
            self.parent_dir, 'tests', 'test_data',
            'fixture_cmf_description_flat_test.json')
        self.cmf = CrashMoveFolder(self.path_to_valid_cmf_des)
        self.event = Event(
            os.path.join(self.cmf.path, 'event_description.json'))
        self.arcmap_runner = arcmap_runner.ArcMapRunner(self.event)

        # 1) insert map
        self.df1 = mock.Mock(name='data_frame1')
        self.df1.name = 'data_frame1'
        self.df1.elementHeight = 19
        self.df1.elementWidth = 17
        # 2) main map
        self.df2 = mock.Mock(name='data_frame2')
        self.df2.name = 'data_frame2'
        self.df2.elementHeight = 100
        self.df2.elementWidth = 200
        # 3) main map (same size different asspect ratio)
        self.df3 = mock.Mock(name='data_frame3')
        self.df3.name = 'data_frame3'
        self.df3.elementHeight = 50
        self.df3.elementWidth = 400
        # 4) main map - identical to df3
        self.df4 = mock.Mock(name='data_frame4')
        self.df4.name = 'data_frame4'
        self.df4.elementHeight = 50
        self.df4.elementWidth = 400
        # 5) main map - widest, but not the largest area
        self.df5 = mock.Mock(name='data_frame5')
        self.df5.name = 'data_frame5'
        self.df5.elementHeight = 10
        self.df5.elementWidth = 500
        # 6) inset map - identical to df1 in every way including the name
        self.df6 = mock.Mock(name='data_frame6')
        self.df6.name = 'data_frame1'
        self.df6.elementHeight = 19
        self.df6.elementWidth = 17
    def setUp(self):
        self.parent_dir = os.path.dirname(
            os.path.dirname(os.path.realpath(__file__)))
        self.cmf_descriptor_path = os.path.join(
            self.parent_dir, 'example', 'cmf_description_flat_test.json')
        self.cmf = CrashMoveFolder(self.cmf_descriptor_path,
                                   verify_on_creation=False)
        self.event_descriptor_path = os.path.join(self.parent_dir, 'example',
                                                  'event_description.json')
        self.event = Event(self.event_descriptor_path)
        # self.cmf = CrashMoveFolder(self.cmf_descriptor_path)
        self.lyr_props = LayerProperties(self.cmf,
                                         '',
                                         verify_on_creation=False)

        self.recipe_descriptor_path = os.path.join(
            self.parent_dir, 'example', 'example_single_map_recipe.json')
        with open(self.recipe_descriptor_path) as json_file:
            self.recipe_def = json.load(json_file)

        self.recipe = MapRecipe(self.recipe_def, self.lyr_props)
Example #17
0
    def test_select_recipes(self):
        # This looks for the MapID in the step.running_msg for the selected maps

        # A list of tuples to test
        # * The param (as entered by a human) to be passed to `get_cookbook_steps`. Includes upper, lower
        #   and mixed case, MapIDs that aren't in the cookbook etc. Can be either a string or a list
        # * A list of the mapIDs that should be created (exactly as in the cookbook file - match case etc)
        # * A list of mapID that should NOT be created (and represent a failure)
        #
        # This is the list of mapID that exist in the relevant cookbook
        #  ['MA001', 'MA002', 'MA003', 'MA004']
        test_cases = [('ma001', ['MA001'], ['MA002', 'MA003', 'MA004']),
                      ('MA004', ['MA004'], ['MA001', 'MA002', 'MA003']),
                      ('Ma004', ['MA004'], ['MA001', 'MA002', 'MA003']),
                      ('MA001', ['MA001'], ['MA002', 'MA003', 'MA004']),
                      (['mA001', 'ma002'], ['MA001',
                                            'MA002'], ['MA003', 'MA004']),
                      ('ma999', [], ['MA001', 'MA002', 'MA003', 'MA004']),
                      (None, ['MA001', 'MA002', 'MA003', 'MA004'], [])]

        cmf = CrashMoveFolder(self.path_to_cmf_file)
        lp = LayerProperties(cmf, 'test', False)
        cb = MapCookbook(cmf, lp, False)

        for mapid_arg, should_create, fail_list in test_cases:
            # print('mapid_arg = {}'.format(mapid_arg))
            # print('should_create = {}'.format(should_create))
            # print('fail_list = {}'.format(fail_list))
            should_create_set = set(should_create)

            selected_recipes = plugin_controller.select_recipes(cb, mapid_arg)
            selected_map_ids = set(
                [recipe.mapnumber.upper() for recipe in selected_recipes])

            # Check that no MapID were selected that shouldn't have been:
            self.assertTrue(selected_map_ids.isdisjoint(fail_list))

            # Check that all of the `selected_ids` match the `should_create` set:
            self.assertEqual(should_create_set, selected_map_ids)
    def __init__(self, cmf, extension, verify_on_creation=True):
        """
        Positional Arguments:
            * cmf: Either a CrashMoveFolder object or a path to a cmf_description.json file. If it
                   is a CrashMoveFolder object and cmf.verify_paths() returns False then an
                   ValueError exception will be rasied.
            * extension: The file extension of the layer_rendering file type. (typically `.lyr` for
                         ESRI Layer files, `'.qml` for QGIS style)

        Optional Named Arguments
            verify_on_creation: True by default. If True then
            `verify_match_with_layer_rendering_dir()` will be called from the constructor.

        """
        try:
            if cmf.verify_paths():
                self.cmf = cmf
            else:
                raise ValueError(
                    'The `cmf` parameter for LayerProperties.__init__() can only accept'
                    'values where the paths verify. eg `cmf.verify_paths() == True`.'
                    'The value passed in this case failed this test')
        except AttributeError:
            self.cmf = CrashMoveFolder(cmf, verify_on_creation=True)

        if len(extension) == 0 or extension.startswith('.'):
            self.extension = extension
        else:
            self.extension = '.{}'.format(extension)

        self.properties = {}  # Dictionary
        self._parse()

        if verify_on_creation:
            msg = self.get_difference_with_other_layer_set(
                self._get_lyr_rendering_names_as_set(),
                self._get_mismatch_with_layer_rendering_message)
            if msg:
                raise ValueError(msg)
    def test_cmf_path_validation(self):

        cmf_partial_fail = os.path.join(
            self.parent_dir, 'tests', 'testfiles',
            'fixture_cmf_description_one_file_and_one_dir_not_valid.json')

        # test validation on creation (for failing case) using default parameters
        self.assertRaises(ValueError, CrashMoveFolder, cmf_partial_fail)
        # force validation on creation, for a json file we know would otherwise fail:
        self.assertRaises(ValueError,
                          CrashMoveFolder,
                          cmf_partial_fail,
                          verify_on_creation=True)
        # test validation is correctly disabled on creation, for a json file we know would otherwise fail:
        test_cmf = CrashMoveFolder(cmf_partial_fail, verify_on_creation=False)
        self.assertIsInstance(test_cmf, CrashMoveFolder)

        # check message included in the ValueError:
        with self.assertRaises(ValueError) as cm:
            test_cmf = CrashMoveFolder(cmf_partial_fail,
                                       verify_on_creation=True)

        if six.PY2:
            self.assertRegexpMatches(str(cm.exception), "map_templates")
            self.assertNotRegexpMatches(str(cm.exception), "original_data")
        else:
            self.assertRegex(str(cm.exception), "map_templates")
            self.assertNotRegex(str(cm.exception), "original_data")

        # create a valid CMF object and then test paths, after creation
        test_cmf_path = os.path.join(self.parent_dir, 'example',
                                     'cmf_description_flat_test.json')
        test_cmf = CrashMoveFolder(test_cmf_path)
        self.assertTrue(test_cmf.verify_paths())
        test_cmf_path = os.path.join(
            self.parent_dir, 'example',
            'cmf_description_relative_paths_test.json')
        test_cmf = CrashMoveFolder(test_cmf_path)
        self.assertTrue(test_cmf.verify_paths())
        test_cmf.active_data = os.path.join(self.parent_dir, 'DOES-NOT-EXIST')
        self.assertFalse(test_cmf.verify_paths())
 def check_cmf_description(self, **kwargs):
     self.cmf = CrashMoveFolder(self.cmf_desc_path)
     return (
         'The Crash Move Folder description file open correctly:\n"{}"\n'.
         format(self.cmf_desc_path))
    def test_verify_with_rendering_files(self):
        # self.fail()

        # load a valid CMF
        test_cmf = CrashMoveFolder(self.path_to_valid_cmf_des)
        # Overwright the Layer Properties file path with
        # mapactionpy_controller\tests\testfiles\fixture_layer_properties_four_layers.json
        test_cmf.layer_properties = os.path.join(
            self.parent_dir, 'tests', 'testfiles',
            'fixture_layer_properties_four_layers.json')

        layer_rendering_test_root = os.path.join(self.parent_dir, 'tests',
                                                 'testfiles',
                                                 'test_layer_rendering')
        test_cmf.layer_rendering

        # 1) Exact match of .lyr files and layer properties
        test_cmf.layer_rendering = os.path.join(layer_rendering_test_root,
                                                'four_files_exact_match')
        lyr_lp = LayerProperties(test_cmf, '.lyr', verify_on_creation=True)
        self.assertFalse(lyr_lp.is_difference_with_layer_rendering_dir())

        qml_lp = LayerProperties(test_cmf, '.qml', verify_on_creation=True)
        self.assertFalse(qml_lp.is_difference_with_layer_rendering_dir())

        # 2) .lyr files which don't have layer properties entries
        test_cmf.layer_rendering = os.path.join(layer_rendering_test_root,
                                                'five_files')
        self.assertRaises(ValueError, LayerProperties, test_cmf, '.lyr')
        self.assertRaises(ValueError, LayerProperties, test_cmf, '.qml')

        # 3) layer properties entries which don't have cooresponding .lyr files.
        test_cmf.layer_rendering = os.path.join(layer_rendering_test_root,
                                                'three_files')
        self.assertRaises(ValueError, LayerProperties, test_cmf, '.lyr')
        self.assertRaises(ValueError, LayerProperties, test_cmf, '.qml')

        # 4) Both 2 & 3 combined
        test_cmf.layer_rendering = os.path.join(layer_rendering_test_root,
                                                'four_files_mis_match')
        self.assertRaises(ValueError, LayerProperties, test_cmf, '.lyr')
        self.assertRaises(ValueError, LayerProperties, test_cmf, '.qml')

        # 6) Overrided validation checks in constructor
        test_cmf.layer_rendering = os.path.join(layer_rendering_test_root,
                                                'four_files_mis_match')
        long_lived_lp = LayerProperties(test_cmf,
                                        '.lyr',
                                        verify_on_creation=False)
        self.assertTrue(long_lived_lp.is_difference_with_layer_rendering_dir())
        long_lived_lp.cmf.layer_rendering = os.path.join(
            layer_rendering_test_root, 'four_files_exact_match')
        self.assertFalse(
            long_lived_lp.is_difference_with_layer_rendering_dir())

        # 7) Same results whether or not the '.' character is included in the file extension
        test_cmf.layer_rendering = os.path.join(layer_rendering_test_root,
                                                'four_files_exact_match')
        # passing example with dot
        lyr_lp = LayerProperties(test_cmf, '.lyr', verify_on_creation=True)
        self.assertFalse(lyr_lp.is_difference_with_layer_rendering_dir())
        # passing example without dot
        lyr_lp = LayerProperties(test_cmf, 'lyr', verify_on_creation=True)
        self.assertFalse(lyr_lp.is_difference_with_layer_rendering_dir())
        # failing example with dot
        test_cmf.layer_rendering = os.path.join(layer_rendering_test_root,
                                                'four_files_mis_match')
        self.assertRaises(ValueError, LayerProperties, test_cmf, '.lyr')
        # failing example without dot
        test_cmf.layer_rendering = os.path.join(layer_rendering_test_root,
                                                'four_files_mis_match')
        self.assertRaises(ValueError, LayerProperties, test_cmf, 'lyr')
    def test_get_task_unique_summary(self):
        dnc_json_path = os.path.join(self.parent_dir, 'example',
                                     'data_naming_convention.json')
        nc = NamingConvention(dnc_json_path)
        cmf = CrashMoveFolder(self.path_to_valid_cmf_des)

        # Three cases
        # 1. Parsable and Valid
        # 2. Parsable and Not Valid
        # 3. Not Parsable
        #
        # key= is the name to test
        # value= is a list of expected results from rendering the mustache elements in `mustache_tmpls`
        test_cases = {
            'lka_admn_ad2_py_s5_unocha_pp_freetext': [
                'lka_admn_ad2_py_s5_unocha_pp_freetext',
                'parsable but not valid',
                ('* "ad2" is a recognised value for the clause "datatheme" found in "03_theme.csv"\n'
                 '* "admn" is a recognised value for the clause "datacat" found in "02_category.csv"\n'
                 '* "freetext" is valid for freetext (as is almost anything)\n'
                 '* "lka" is a recognised value for the clause "geoext" found in "01_geoextent.csv"\n'
                 '* "pp" is a recognised value for the clause "perm" found in "07_permission.csv"\n'
                 '* "py" is a recognised value for the clause "geom" found in "04_geometry.csv"\n'
                 '* "s5" is a recognised value for the clause "scale" found in "05_scale.csv"\n'
                 '* "unocha" is a recognised value for the clause "source" found in "06_source.csv"\n'
                 ), ''
            ],
            'lka_admn_ad77_py_s5_unocha_pp_freetext': [
                'lka_admn_ad77_py_s5_unocha_pp_freetext',
                'parsable but not valid',
                ('* "admn" is a recognised value for the clause "datacat" found in "02_category.csv"\n'
                 '* "freetext" is valid for freetext (as is almost anything)\n'
                 '* "lka" is a recognised value for the clause "geoext" found in "01_geoextent.csv"\n'
                 '* "pp" is a recognised value for the clause "perm" found in "07_permission.csv"\n'
                 '* "py" is a recognised value for the clause "geom" found in "04_geometry.csv"\n'
                 '* "s5" is a recognised value for the clause "scale" found in "05_scale.csv"\n'
                 '* "unocha" is a recognised value for the clause "source" found in "06_source.csv"\n'
                 ),
                '* "ad77" is not a recognised value for the clause "datatheme" found in "03_theme.csv"\n'
            ],
            'unparsable-name': ['unparsable-name', 'not parsable', '', '']
        }

        mustache_tmpls = [
            '<%name_result.name_to_validate%>',
            ('<%#name_result.is_parsable%>parsable but not valid<%/name_result.is_parsable%>'
             '<%^name_result.is_parsable%>not parsable<%/name_result.is_parsable%>'
             ),
            '<%#name_result.valid_clause_list%>* <%&valid_clause%>\n<%/name_result.valid_clause_list%>',
            '<%#name_result.invalid_clause_list %>* <%&invalid_clause%>\n<%/name_result.invalid_clause_list %>'
        ]

        # for name_to_test, expected_result_list in test_cases.iteritems():
        while test_cases:
            name_to_test, expected_result_list = test_cases.popitem()
            # print(name_to_test)
            nr = nc.validate(name_to_test)
            fdnt = FixDataNameTask(nr, cmf)

            for test_template, expected_result in zip(mustache_tmpls,
                                                      expected_result_list):
                # create the Task object
                # override the unique identifier for test purposes
                fdnt._primary_key_template = test_template

                actual_result = fdnt.get_task_unique_summary()
                self.assertEqual(actual_result, expected_result)
Example #23
0
class BaseRunnerPlugin(object):
    def __init__(self, hum_event, **kwargs):
        self.hum_event = hum_event
        self.cmf = CrashMoveFolder(self.hum_event.cmf_descriptor_path)

        if not self.cmf.verify_paths():
            raise ValueError(
                "Cannot find paths and directories referenced by cmf {}".
                format(self.cmf.path))

        if self.__class__ is BaseRunnerPlugin:
            raise NotImplementedError(
                'BaseRunnerPlugin is an abstract class and cannot be instantiated directly'
            )

    def get_projectfile_extension(self, **kwargs):
        raise NotImplementedError(
            'BaseRunnerPlugin is an abstract class and the `get_projectfile_extension`'
            ' method cannot be called directly')

    def get_lyr_render_extension(self, **kwargs):
        raise NotImplementedError(
            'BaseRunnerPlugin is an abstract class and the `get_lyr_render_extension`'
            ' method cannot be called directly')

    def _get_all_templates_by_regex(self, recipe):
        """
        Gets the fully qualified filenames of map templates, which exist in `self.cmf.map_templates` whose
        filenames match the regex `recipe.template`.

        @param recipe: A MapRecipe object.
        @returns: A list of all of the templates, stored in `cmf.map_templates` whose
                 filename matches the regex `recipe.template` and that have the extention
                 `self.get_projectfile_extension()`
        """
        def _is_relevant_file(f):
            extension = os.path.splitext(f)[1]
            logger.debug(
                'checking file "{}", with extension "{}", against pattern "{}" and "{}"'
                .format(f, extension, recipe.template,
                        self.get_projectfile_extension()))
            if re.search(recipe.template, f):
                logger.debug('file {} matched regex'.format(f))
                f_path = os.path.join(self.cmf.map_templates, f)
                logger.debug(
                    'file {} joined with self.cmf.map_templates "{}"'.format(
                        f, f_path))
                return (os.path.isfile(f_path)) and (
                    extension == self.get_projectfile_extension())
            else:
                return False

        # TODO: This results in calling `os.path.join` twice for certain files
        logger.debug('searching for map templates in; {}'.format(
            self.cmf.map_templates))
        all_filenames = os.listdir(self.cmf.map_templates)
        logger.debug('all available template files:\n\t{}'.format(
            '\n\t'.join(all_filenames)))
        relevant_filenames = [
            os.path.realpath(os.path.join(self.cmf.map_templates, fi))
            for fi in all_filenames if _is_relevant_file(fi)
        ]
        logger.debug('possible template files:\n\t{}'.format(
            '\n\t'.join(relevant_filenames)))
        return relevant_filenames

    def _get_template_by_aspect_ratio(self, template_aspect_ratios, target_ar):
        """
        Selects the template which best matches the required aspect ratio.

        @param possible_aspect_ratios: A list of tuples. For each tuple the first element is the path to the
                                       template. The second element is the relevant aspect ratio for that
                                       template. Typically this would be generated by
                                       `get_aspect_ratios_of_templates()`.
        @param target_ar: The taret aspect ratio - typically the aspect ratio of the bounding box for the country
                          being mapped.
        @returns: The path of the template with the best matching aspect ratio.
        """
        logger.info(
            'Selecting from available templates based on the most best matching aspect ratio'
        )

        # Target is more landscape than the most landscape template
        most_landscape = max(template_aspect_ratios, key=itemgetter(1))
        if most_landscape[1] < target_ar:
            logger.info(
                'Target area of interest is more landscape than the most landscape template'
            )
            return most_landscape[0]

        # Target is more portrait than the most portrait template
        most_portrait = min(template_aspect_ratios, key=itemgetter(1))
        if most_portrait[1] > target_ar:
            logger.info(
                'Target area of interest is more portrait than the most portrait template'
            )
            return most_portrait[0]

        # The option with the smallest aspect ratio that is larger than target_ar
        larger_ar = min([(templ_path, templ_ar)
                         for templ_path, templ_ar in template_aspect_ratios
                         if templ_ar >= target_ar],
                        key=itemgetter(1))
        # The option with the largest aspect ratio that is smaller than target_ar
        smaller_ar = max([(templ_path, templ_ar)
                          for templ_path, templ_ar in template_aspect_ratios
                          if templ_ar <= target_ar],
                         key=itemgetter(1))

        # Linear combination:
        # if (2*target_ar) > (larger_ar[1] + smaller_ar[1]):
        #     return larger_ar[0]

        # asmith: personally I think that this is the better option, but will go with the linear combination for now
        # logarithmic combination
        if (2 * math.log(target_ar)) > (math.log(larger_ar[1]) +
                                        math.log(smaller_ar[1])):
            logger.info(
                'Aspect ratio of the target area of interest lies between the aspect ratios of the'
                ' available templates')
            return larger_ar[0]

        return smaller_ar[0]

    def get_aspect_ratios_of_templates(self, possible_templates, recipe):
        """
        Plugins are required to implement this method.

        The implementation should calculate the aspect ratio of the principal map frame within the list of
        templates. The definition of "principal" is left to the plugin, though is typically the largest map
        frame.

        @param possible_templates: A list of paths to possible templates
        @returns: A list of tuples. For each tuple the first element is the path to the template. The second
                  element is the aspect ratio of the largest* map frame within that template.
                  See `_get_largest_map_frame` for the description of how largest is determined.
        @raises NotImplementedError: In the base class.
        """
        raise NotImplementedError(
            'BaseRunnerPlugin is an abstract class and the `get_aspect_ratios_of_templates`'
            ' method cannot be called directly')

    def _get_aspect_ratio_of_bounds(self, bounds):
        minx, miny, maxx, maxy = bounds
        dx = (
            maxx - minx
        ) % 360  # Accounts for the case where the bounds stradles the 180 meridian
        dy = maxy - miny

        return float(dx) / dy

    def get_templates(self, **kwargs):
        """
        Updates the recipe's `template_path` value. The result is the absolute path to the template.

        To select the appropriate template it uses two inputs.
        * The `recipe.template` value, which is a regex for the filename of the possible templates
        * The target asspect ratio. If the aspect ratio of the target data can be determined then this is
          also used to select the best matching template, amogst those which match the regex. If the
          target ratio cannot be determined fromsource gis data, then the target ratio of 1.0 will be
          used.
        """
        recipe = kwargs['state']
        # If there already is a valid `recipe.map_project_path` just skip with method
        if recipe.map_project_path:
            if os.path.exists(recipe.map_project_path):
                return recipe
            else:
                raise ValueError(
                    'Unable to locate map project file: {}'.format(
                        recipe.map_project_path))

        # use `recipe.template` as regex to locate one or more templates
        possible_templates = self._get_all_templates_by_regex(recipe)

        # Select the template with the most appropriate aspect ratio
        possible_aspect_ratios = self.get_aspect_ratios_of_templates(
            possible_templates, recipe)

        mf = recipe.get_frame(recipe.principal_map_frame)
        # Default value
        target_aspect_ratio = 1.0
        # If the MapFrame's target extent is not None, then use that:
        if mf.extent:
            target_aspect_ratio = self._get_aspect_ratio_of_bounds(mf.extent)

        # use logic to workout which template has best aspect ratio
        # obviously not this logic though:
        recipe.template_path = self._get_template_by_aspect_ratio(
            possible_aspect_ratios, target_aspect_ratio)

        # TODO re-enable "Have the input files changed?"
        # Have the input shapefiles changed?
        return recipe

    # TODO: asmith 2020/03/03
    # 1) Please avoid hardcoding the naming convention for the mxds wherever possible. The Naming Convention
    # classes can avoid the need to hardcode the naming convention for the input mxd templates. It might be
    # possible to avoid the need to hardcode the naming convention for the output mxds using a
    # String.Template be specified within the Cookbook?
    # https://docs.python.org/2/library/string.html#formatspec
    # https://www.python.org/dev/peps/pep-3101/
    #
    # 2) This only checks the filename for the mxd - it doesn't check the values within the text element of
    # the map layout view (and hence the output metadata).
    def get_next_map_version_number(self, mapNumberDirectory, mapNumber,
                                    mapFileName):
        versionNumber = 0
        files = glob.glob(mapNumberDirectory + os.sep + mapNumber +
                          '-v[0-9][0-9]-' + mapFileName + '.mxd')
        for file in files:
            versionNumber = int(
                os.path.basename(file).replace(mapNumber + '-v', '').replace(
                    ('-' + mapFileName + '.mxd'), ''))  # noqa
        versionNumber = versionNumber + 1
        return versionNumber

    # TODO Is it possible to aviod the need to hardcode the naming convention for the output mxds? Eg could a
    # String.Template be specified within the Cookbook?
    # https://docs.python.org/2/library/string.html#formatspec
    # https://www.python.org/dev/peps/pep-3101/
    def create_ouput_map_project(self, **kwargs):
        recipe = kwargs['state']
        # Create `mapNumberDirectory` for output
        output_dir = os.path.join(self.cmf.map_projects, recipe.mapnumber)

        if not (os.path.isdir(output_dir)):
            os.mkdir(output_dir)

        # Construct output MXD/QPRJ name
        logger.debug(
            'About to create new map project file for product "{}"'.format(
                recipe.product))
        output_map_base = slugify(recipe.product)
        logger.debug('Set output name for new map project file to "{}"'.format(
            output_map_base))
        recipe.version_num = self.get_next_map_version_number(
            output_dir, recipe.mapnumber, output_map_base)
        output_map_name = '{}-v{}-{}{}'.format(
            recipe.mapnumber,
            str(recipe.version_num).zfill(2), output_map_base,
            self.get_projectfile_extension())
        recipe.map_project_path = os.path.abspath(
            os.path.join(output_dir, output_map_name))
        logger.debug('Path for new map project file; {}'.format(
            recipe.map_project_path))
        logger.debug('Map Version number; {}'.format(recipe.version_num))

        # Copy `src_template` to `recipe.map_project_path`
        copyfile(recipe.template_path, recipe.map_project_path)

        return recipe

    def export_maps(self, **kwargs):
        """
        Generates all file for export.

        Accumulate some of the parameters for export XML, then calls
        _do_export(....) to do that actual work
        """
        recipe = kwargs['state']
        export_params = {}
        properties = {}  # For properties from MapAction Toolbox
        try:
            export_params = self._create_export_dir(export_params, recipe)
            if 'properties' in kwargs:
                properties = kwargs['properties']
                for key in list(properties.keys()):
                    export_params[str(key)] = properties[str(key)]
            export_params = self._do_export(export_params, recipe)
        except Exception as exp:
            logger.error(
                'Failed to export the map. export_params = "{}"'.format(
                    export_params))
            logger.error(exp)

        self.zip_exported_files(export_params)

    def _create_export_dir(self, export_params, recipe):
        # Accumulate parameters for export XML
        version_str = "v" + str(recipe.version_num).zfill(2)
        export_directory = os.path.abspath(
            os.path.join(self.cmf.export_dir, recipe.mapnumber, version_str))
        export_params["exportDirectory"] = export_directory

        try:
            os.makedirs(export_directory)
        except OSError as exc:  # Python >2.5
            # Note 'errno.EEXIST' is not a typo. There should be two 'E's.
            # https://docs.python.org/2/library/errno.html#errno.EEXIST
            if exc.errno == errno.EEXIST and os.path.isdir(export_directory):
                pass
            else:
                raise

        return export_params

    def _do_export(self, export_params, recipe):
        raise NotImplementedError(
            'BaseRunnerPlugin is an abstract class and the `export_maps`'
            ' method cannot be called directly')

    def zip_exported_files(self, export_params):
        # Get key params as local variables
        core_file_name = export_params['coreFileName']
        export_dir = export_params['exportDirectory']
        mdr_xml_file_path = export_params['exportXmlFileLocation']
        jpg_path = export_params['jpgFileLocation']
        png_thumbnail_path = export_params['pngThumbNailFileLocation']
        # And now Zip
        zipFileName = core_file_name + ".zip"
        zipFileLocation = os.path.join(export_dir, zipFileName)

        with ZipFile(zipFileLocation, 'w') as zipObj:
            zipObj.write(mdr_xml_file_path,
                         os.path.basename(mdr_xml_file_path))
            zipObj.write(jpg_path, os.path.basename(jpg_path))
            zipObj.write(png_thumbnail_path,
                         os.path.basename(png_thumbnail_path))
            if (len(export_params.get("emfFileLocation", "")) > 0):
                zipObj.write(
                    export_params['emfFileLocation'],
                    os.path.basename(export_params['emfFileLocation']))

            # TODO: asmith 2020/03/03
            # Given we are explictly setting the pdfFileName for each page within the DDPs
            # it is possible return a list of all of the filenames for all of the PDFs. Please
            # can we use this list to include in the zip file. There are edge cases where just
            # adding all of the pdfs in a particular directory might not behave correctly (eg if
            # the previous run had crashed midway for some reason)
            for pdf in os.listdir(export_dir):
                if pdf.endswith(".pdf"):
                    zipObj.write(
                        os.path.join(export_dir, pdf),
                        os.path.basename(os.path.join(export_dir, pdf)))
        print("Export complete to " + export_dir)

    def build_project_files(self, **kwargs):
        raise NotImplementedError(
            'BaseRunnerPlugin is an abstract class and the `build_project_files`'
            ' method cannot be called directly')
 def test_cmf_description_adapter(self):
     test_cmf = CrashMoveFolder(self.path_to_valid_cmf_des)
     test_cd = task_renderer.cmf_description_adapter(test_cmf)
     self.assertEqual(self.dir_to_valid_cmf_des, test_cd['cmf']['path'])
Example #25
0
    def test_layer_data_schema(self):

        null_schema = True

        passing_schema = yaml.safe_load(r"""
required:
    - name_en
properties:
    geometry_type:
        items:
            enum:
                - MultiPolygon
                - Polygon
        additionalItems: false
    crs:
        items:
            enum:
                - EPSG:2090
        additionalItems: false
""")

        # Note the missing `enum` blocks
        failing_schema = yaml.safe_load(r"""
required:
    - name_en
properties:
    geometry_type:
        items:
            - MultiPolygon
            - Polygon
        additionalItems: false
    crs:
        items:
            - EPSG:2090
        additionalItems: false
""")

        cmf = CrashMoveFolder(
            os.path.join(self.parent_dir, 'example', 'cmf_description_relative_paths_test.json'))
        cmf.layer_properties = os.path.join(
            self.parent_dir, 'tests', 'testfiles', 'cookbooks', 'fixture_layer_properties_for_atlas.json'
        )

        # Two cases where data schema is valid yaml
        for test_schema in [null_schema, passing_schema]:
            with mock.patch('mapactionpy_controller.data_schemas.yaml.safe_load') as mock_safe_load:
                mock_safe_load.return_value = test_schema
                test_lp = LayerProperties(cmf, ".lyr", verify_on_creation=False)

                MapRecipe(fixtures.recipe_with_positive_iso3_code, test_lp)
                self.assertTrue(True, 'validated jsonschema')

        # case where data schema file itself malformed somehow
        with mock.patch('mapactionpy_controller.data_schemas.yaml.safe_load') as mock_safe_load:
            mock_safe_load.return_value = failing_schema

            self.assertRaises(
                jsonschema.exceptions.SchemaError,
                MapRecipe,
                fixtures.recipe_with_positive_iso3_code,
                test_lp
            )
Example #26
0
    def test_get_map_frame_extents(self):
        cmf = CrashMoveFolder(
            os.path.join(self.parent_dir, 'example',
                         'cmf_description_relative_paths_test.json'))
        cmf.layer_properties = os.path.join(
            self.parent_dir, 'tests', 'testfiles', 'cookbooks',
            'fixture_layer_properties_for_atlas.json')
        test_lp = LayerProperties(cmf, ".lyr", verify_on_creation=False)

        # recipe with layer name only
        with open(
                os.path.join(
                    self.parent_dir, 'tests', 'testfiles',
                    'fixture_cookbook_1map_5layers_1frame.json')) as rf:
            cookbook_def = json.load(rf)
        # get the first (only) recipe in the cookbook
        recipe_def = cookbook_def['recipes'].pop()

        generic_lyr_def = json.loads('''{
            "name": "the_name",
            "reg_exp": "^wrl_admn_ad0_py_(.*?)_(.*?)_([phm][phm])(.+)shp$",
            "schema_definition": "admin1_reference.yml",
            "definition_query": "",
            "display": true,
            "add_to_legend": true,
            "label_classes": []
        }''')

        # Case 1
        # One or more layers does not have it's extent defined
        case1_list = [('case1_lyrA', (35, 33, 36, 34), 'epsg:4326'),
                      ('case1_lyrB', None, None)]
        case1_result = (35, 33, 36, 34)

        # Case 2
        # Simple union with two lyrs of same crs
        case2_list = [('case2_lyrA', (33, 51, 36, 58), 'epsg:4326'),
                      ('case2_lyrB', (15, 52, 35, 55), 'epsg:4326')]
        case2_result = (15, 51, 36, 58)

        # Case 3
        # Union with two lyrs of with different crs
        # 'epsg:4326'== WGS1984, 'epsg:3785' == Web Mercator
        case3_list = [('case3_lyrA', (33, 51, 36, 58), 'epsg:4326'),
                      ('case3_lyrB', (1669792.36, 6800125.45, 3896182.18,
                                      7361866.11), 'epsg:3785')]
        case3_result = (15, 51, 36, 58)

        # Case 4
        # One layer which stradles 180 degree meridian

        all_test_params = [(case1_list, case1_result),
                           (case2_list, case2_result),
                           (case3_list, case3_result)]

        for test_list, expected_result in all_test_params:
            test_recipe = MapRecipe(recipe_def, test_lp)

            # Build up a mock list of layer to test
            replacement_lyrs = []
            for name, extent, crs in test_list:
                new_lyr = RecipeLayer(generic_lyr_def,
                                      test_lp,
                                      verify_on_creation=False)
                new_lyr.name = name
                new_lyr.use_for_frame_extent = bool(extent)
                # vaugely near Lebanon
                new_lyr.extent = extent
                new_lyr.crs = crs

                replacement_lyrs.append(new_lyr)

            # print('test_get_map_frame_extents')
            # print('expected_result = {}'.format(expected_result))
            test_frame = test_recipe.map_frames.pop()
            test_frame.crs = 'epsg:4326'
            test_frame.layers = replacement_lyrs
            test_frame.calc_extent(state=test_recipe)
            actual_result = test_frame.extent
            # print('actual_result = {}'.format(actual_result))
            for actual, expected in zip(actual_result, expected_result):
                self.assertAlmostEqual(actual, expected)
Example #27
0
    def test_filter_lyr_for_use_in_frame_extent(self):
        # Have included the digit at the start of the string, so that can be sorted easily.
        cmf = CrashMoveFolder(
            os.path.join(self.parent_dir, 'example',
                         'cmf_description_relative_paths_test.json'))
        cmf.layer_properties = os.path.join(
            self.parent_dir, 'tests', 'testfiles', 'cookbooks',
            'fixture_layer_properties_for_atlas.json')
        test_lp = LayerProperties(cmf, ".lyr", verify_on_creation=False)

        # recipe with layer name only
        with open(
                os.path.join(
                    self.parent_dir, 'tests', 'testfiles',
                    'fixture_cookbook_1map_5layers_1frame.json')) as rf:
            cookbook_def = json.load(rf)
        # get the first (only) recipe in the cookbook
        recipe_def = cookbook_def['recipes'].pop()

        generic_lyr_def = json.loads('''{
            "name": "the_name",
            "reg_exp": "^wrl_admn_ad0_py_(.*?)_(.*?)_([phm][phm])(.+)shp$",
            "schema_definition": "admin1_reference.yml",
            "definition_query": "",
            "display": true,
            "add_to_legend": true,
            "label_classes": []
        }''')

        # Case 1
        # test white list
        test_white_list1 = [('1a', True), ('1b', False), ('1c', None),
                            ('1d', True), ('1e', False)]
        expected_white_result1 = ['1a', '1d']

        test_white_list2 = [('2a', True), ('2b', None), ('2c', None),
                            ('2d', True), ('2e', None)]
        expected_white_result2 = ['2a', '2d']

        # Case 2
        # Black List
        test_black_list = [('3a', None), ('3b', False), ('3c', None),
                           ('3d', None), ('3e', False)]
        expected_black_result = ['3a', '3c', '3d']

        # Case 3
        # Default
        test_default_list = [('4a', None), ('4b', None), ('4c', None),
                             ('4d', None), ('4e', None)]
        expected_default_result = ['4a', '4b', '4c', '4d', '4e']

        all_test_params = [(test_white_list1, expected_white_result1),
                           (test_white_list2, expected_white_result2),
                           (test_black_list, expected_black_result),
                           (test_default_list, expected_default_result)]

        for test_list, expected_result in all_test_params:
            test_recipe = MapRecipe(recipe_def, test_lp)

            # Build up a mock list of layer to test
            replacement_lyrs = []
            for test_lyr_details in test_list:
                new_lyr = RecipeLayer(generic_lyr_def,
                                      test_lp,
                                      verify_on_creation=False)
                new_lyr.name = test_lyr_details[0]
                new_lyr.use_for_frame_extent = test_lyr_details[1]
                # vaugely near Lebanon
                new_lyr.extent = (35, 33, 36, 34)
                new_lyr.crs = 'epsg:4326'

                replacement_lyrs.append(new_lyr)

            test_frame = test_recipe.map_frames.pop()
            test_frame.layers = replacement_lyrs
            result_lyrs = test_frame._filter_lyr_for_use_in_frame_extent()
            actual_result = [lyr.name for lyr in result_lyrs]
            self.assertEqual(actual_result, expected_result)