def test_map_common_versionsuffixes(self): """Test mapping between two toolchain hierarchies""" test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') init_config(build_options={ 'robot_path': [test_easyconfigs], 'silent': True, 'valid_module_classes': module_classes(), }) get_toolchain_hierarchy.clear() gcc_binutils_tc = {'name': 'GCC', 'version': '4.9.3-2.26'} iccifort_binutils_tc = {'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'} toolchain_mapping = map_toolchain_hierarchies(iccifort_binutils_tc, gcc_binutils_tc, self.modtool) possible_mappings = map_common_versionsuffixes('binutils', iccifort_binutils_tc, toolchain_mapping) self.assertEqual(possible_mappings, {'-binutils-2.25': '-binutils-2.26'}) # Make sure we only map upwards, here it's gzip 1.4 in gcc and 1.6 in iccifort possible_mappings = map_common_versionsuffixes('gzip', iccifort_binutils_tc, toolchain_mapping) self.assertEqual(possible_mappings, {}) # newer gzip is picked up other way around (GCC -> iccifort) toolchain_mapping = map_toolchain_hierarchies(gcc_binutils_tc, iccifort_binutils_tc, self.modtool) possible_mappings = map_common_versionsuffixes('gzip', gcc_binutils_tc, toolchain_mapping) self.assertEqual(possible_mappings, {'-gzip-1.4': '-gzip-1.6'})
def test_map_easyconfig_to_target_tc_hierarchy(self): """Test mapping of easyconfig to target hierarchy""" test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') init_config(build_options={ 'robot_path': test_easyconfigs, 'silent': True, 'valid_module_classes': module_classes(), }) get_toolchain_hierarchy.clear() gcc_binutils_tc = {'name': 'GCC', 'version': '4.9.3-2.26'} iccifort_binutils_tc = {'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'} # The below mapping includes a binutils mapping (2.26 to 2.25) tc_mapping = map_toolchain_hierarchies(gcc_binutils_tc, iccifort_binutils_tc, self.modtool) ec_spec = os.path.join(test_easyconfigs, 'h', 'hwloc', 'hwloc-1.6.2-GCC-4.9.3-2.26.eb') tweaked_spec = map_easyconfig_to_target_tc_hierarchy(ec_spec, tc_mapping) tweaked_ec = process_easyconfig(tweaked_spec)[0] tweaked_dict = tweaked_ec['ec'].asdict() # First check the mapped toolchain key, value = 'toolchain', iccifort_binutils_tc self.assertTrue(key in tweaked_dict and value == tweaked_dict[key]) # Also check that binutils has been mapped for key, value in {'name': 'binutils', 'version': '2.25', 'versionsuffix': ''}.items(): self.assertTrue(key in tweaked_dict['builddependencies'][0] and value == tweaked_dict['builddependencies'][0][key])
def test_map_toolchain_hierarchies(self): """Test mapping between two toolchain hierarchies""" test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') init_config(build_options={ 'robot_path': test_easyconfigs, 'silent': True, 'valid_module_classes': module_classes(), }) get_toolchain_hierarchy.clear() foss_tc = {'name': 'foss', 'version': '2018a'} gompi_tc = {'name': 'gompi', 'version': '2018a'} iimpi_tc = {'name': 'iimpi', 'version': '2016.01'} # GCCcore is mapped to GCC, iccifort is mapped to GCC, iimpi is mapped to gompi expected = { 'GCCcore': {'name': 'GCC', 'version': '6.4.0-2.28'}, 'iccifort': {'name': 'GCC', 'version': '6.4.0-2.28'}, 'iimpi': {'name': 'gompi', 'version': '2018a'}, } self.assertEqual(map_toolchain_hierarchies(iimpi_tc, foss_tc, self.modtool), expected) # GCC is mapped to iccifort, gompi is mapped to iimpi expected = { 'GCC': {'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'}, 'gompi': {'name': 'iimpi', 'version': '2016.01'} } self.assertEqual(map_toolchain_hierarchies(gompi_tc, iimpi_tc, self.modtool), expected) # Expect an error when there is no possible mapping error_msg = "No possible mapping from source toolchain spec .*" self.assertErrorRegex(EasyBuildError, error_msg, map_toolchain_hierarchies, foss_tc, iimpi_tc, self.modtool) # Test that we correctly include GCCcore binutils when it is there gcc_binutils_tc = {'name': 'GCC', 'version': '4.9.3-2.26'} iccifort_binutils_tc = {'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'} # Should see a binutils in the mapping (2.26 will get mapped to 2.25) expected = { 'GCC': {'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'}, 'GCCcore': {'name': 'GCCcore', 'version': '4.9.3'}, 'binutils': {'version': '2.25', 'versionsuffix': ''} } self.assertEqual(map_toolchain_hierarchies(gcc_binutils_tc, iccifort_binutils_tc, self.modtool), expected)
def test_map_toolchain_hierarchies(self): """Test mapping between two toolchain hierarchies""" test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') init_config(build_options={ 'robot_path': test_easyconfigs, 'silent': True, 'valid_module_classes': module_classes(), }) get_toolchain_hierarchy.clear() foss_tc = {'name': 'foss', 'version': '2018a'} gompi_tc = {'name': 'gompi', 'version': '2018a'} iimpi_tc = {'name': 'iimpi', 'version': '2016.01'} # GCCcore is mapped to GCC, iccifort is mapped to GCC, iimpi is mapped to gompi expected = { 'GCCcore': {'name': 'GCC', 'version': '6.4.0-2.28'}, 'iccifort': {'name': 'GCC', 'version': '6.4.0-2.28'}, 'iimpi': {'name': 'gompi', 'version': '2018a'}, } self.assertEqual(map_toolchain_hierarchies(iimpi_tc, foss_tc, self.modtool), expected) # GCC is mapped to iccifort, gompi is mapped to iimpi expected = { 'GCC': {'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'}, 'gompi': {'name': 'iimpi', 'version': '2016.01'} } self.assertEqual(map_toolchain_hierarchies(gompi_tc, iimpi_tc, self.modtool), expected) # Expect an error when there is no possible mapping error_msg = "No possible mapping from source toolchain spec .*" self.assertErrorRegex(EasyBuildError, error_msg, map_toolchain_hierarchies, foss_tc, iimpi_tc, self.modtool) # Test that we correctly include GCCcore binutils when it is there gcc_binutils_tc = {'name': 'GCC', 'version': '4.9.3-2.26'} iccifort_binutils_tc = {'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'} # Should see a binutils in the mapping (2.26 will get mapped to 2.25) expected = { 'GCC': {'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'}, 'GCCcore': {'name': 'GCCcore', 'version': '4.9.3'}, 'binutils': {'version': '2.25', 'versionsuffix': ''} } self.assertEqual(map_toolchain_hierarchies(gcc_binutils_tc, iccifort_binutils_tc, self.modtool), expected)
def test_find_potential_version_mappings(self): """Test ability to find potential version mappings of a dependency for a given toolchain mapping""" test_easyconfigs = os.path.join( os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') init_config( build_options={ 'robot_path': [test_easyconfigs], 'silent': True, 'valid_module_classes': module_classes(), }) get_toolchain_hierarchy.clear() gcc_binutils_tc = {'name': 'GCC', 'version': '4.9.3-2.26'} iccifort_binutils_tc = { 'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25' } # The below mapping includes a binutils mapping (2.26 to 2.25) tc_mapping = map_toolchain_hierarchies(gcc_binutils_tc, iccifort_binutils_tc, self.modtool) ec_spec = os.path.join(test_easyconfigs, 'h', 'hwloc', 'hwloc-1.6.2-GCC-4.9.3-2.26.eb') parsed_ec = process_easyconfig(ec_spec)[0] gzip_dep = [ dep for dep in parsed_ec['ec']['dependencies'] if dep['name'] == 'gzip' ][0] self.assertEqual(gzip_dep['full_mod_name'], 'gzip/1.4-GCC-4.9.3-2.26') potential_versions = find_potential_version_mappings( gzip_dep, tc_mapping) self.assertEqual(len(potential_versions), 1) # Should see version 1.6 of gzip with iccifort toolchain expected = { 'path': os.path.join(test_easyconfigs, 'g', 'gzip', 'gzip-1.6-iccifort-2016.1.150-GCC-4.9.3-2.25.eb'), 'toolchain': { 'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25' }, 'version': '1.6', } self.assertEqual(potential_versions[0], expected)
def test_map_easyconfig_to_target_tc_hierarchy(self): """Test mapping of easyconfig to target hierarchy""" test_easyconfigs = os.path.join( os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') init_config( build_options={ 'robot_path': test_easyconfigs, 'silent': True, 'valid_module_classes': module_classes(), }) get_toolchain_hierarchy.clear() gcc_binutils_tc = {'name': 'GCC', 'version': '4.9.3-2.26'} iccifort_binutils_tc = { 'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25' } # The below mapping includes a binutils mapping (2.26 to 2.25) tc_mapping = map_toolchain_hierarchies(gcc_binutils_tc, iccifort_binutils_tc, self.modtool) ec_spec = os.path.join(test_easyconfigs, 'h', 'hwloc', 'hwloc-1.6.2-GCC-4.9.3-2.26.eb') tweaked_spec = map_easyconfig_to_target_tc_hierarchy( ec_spec, tc_mapping) tweaked_ec = process_easyconfig(tweaked_spec)[0] tweaked_dict = tweaked_ec['ec'].asdict() # First check the mapped toolchain key, value = 'toolchain', iccifort_binutils_tc self.assertTrue(key in tweaked_dict and value == tweaked_dict[key]) # Also check that binutils has been mapped for key, value in { 'name': 'binutils', 'version': '2.25', 'versionsuffix': '' }.items(): self.assertTrue( key in tweaked_dict['builddependencies'][0] and value == tweaked_dict['builddependencies'][0][key])
def test_map_easyconfig_to_target_tc_hierarchy(self): """Test mapping of easyconfig to target hierarchy""" test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') build_options = { 'robot_path': [test_easyconfigs], 'silent': True, 'valid_module_classes': module_classes(), } init_config(build_options=build_options) get_toolchain_hierarchy.clear() gcc_binutils_tc = {'name': 'GCC', 'version': '4.9.3-2.26'} iccifort_binutils_tc = {'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'} # The below mapping includes a binutils mapping (2.26 to 2.25) tc_mapping = map_toolchain_hierarchies(gcc_binutils_tc, iccifort_binutils_tc, self.modtool) ec_spec = os.path.join(test_easyconfigs, 'h', 'hwloc', 'hwloc-1.6.2-GCC-4.9.3-2.26.eb') tweaked_spec = map_easyconfig_to_target_tc_hierarchy(ec_spec, tc_mapping) tweaked_ec = process_easyconfig(tweaked_spec)[0] tweaked_dict = tweaked_ec['ec'].asdict() # First check the mapped toolchain key, value = 'toolchain', iccifort_binutils_tc self.assertTrue(key in tweaked_dict and value == tweaked_dict[key]) # Also check that binutils has been mapped for key, value in {'name': 'binutils', 'version': '2.25', 'versionsuffix': ''}.items(): self.assertTrue(key in tweaked_dict['builddependencies'][0] and value == tweaked_dict['builddependencies'][0][key]) # Now test the case where we try to update the dependencies init_config(build_options=build_options) get_toolchain_hierarchy.clear() tweaked_spec = map_easyconfig_to_target_tc_hierarchy(ec_spec, tc_mapping, update_dep_versions=True) tweaked_ec = process_easyconfig(tweaked_spec)[0] tweaked_dict = tweaked_ec['ec'].asdict() # First check the mapped toolchain key, value = 'toolchain', iccifort_binutils_tc self.assertTrue(key in tweaked_dict and value == tweaked_dict[key]) # Also check that binutils has been mapped for key, value in {'name': 'binutils', 'version': '2.25', 'versionsuffix': ''}.items(): self.assertTrue( key in tweaked_dict['builddependencies'][0] and value == tweaked_dict['builddependencies'][0][key] ) # Also check that the gzip dependency was upgraded for key, value in {'name': 'gzip', 'version': '1.6', 'versionsuffix': ''}.items(): self.assertTrue(key in tweaked_dict['dependencies'][0] and value == tweaked_dict['dependencies'][0][key]) # Make sure there are checksums for our next test self.assertTrue(tweaked_dict['checksums']) # Test the case where we also update the software version at the same time init_config(build_options=build_options) get_toolchain_hierarchy.clear() new_version = '1.x.3' tweaked_spec = map_easyconfig_to_target_tc_hierarchy(ec_spec, tc_mapping, update_build_specs={'version': new_version}, update_dep_versions=True) tweaked_ec = process_easyconfig(tweaked_spec)[0] tweaked_dict = tweaked_ec['ec'].asdict() # First check the mapped toolchain key, value = 'toolchain', iccifort_binutils_tc self.assertTrue(key in tweaked_dict and value == tweaked_dict[key]) # Also check that binutils has been mapped for key, value in {'name': 'binutils', 'version': '2.25', 'versionsuffix': ''}.items(): self.assertTrue( key in tweaked_dict['builddependencies'][0] and value == tweaked_dict['builddependencies'][0][key] ) # Also check that the gzip dependency was upgraded for key, value in {'name': 'gzip', 'version': '1.6', 'versionsuffix': ''}.items(): self.assertTrue(key in tweaked_dict['dependencies'][0] and value == tweaked_dict['dependencies'][0][key]) # Finally check that the version was upgraded key, value = 'version', new_version self.assertTrue(key in tweaked_dict and value == tweaked_dict[key]) # and that the checksum was removed self.assertFalse(tweaked_dict['checksums']) # Check that if we update a software version, it also updates the version if the software appears in an # extension list (like for a PythonBundle) ec_spec = os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') # Create the trivial toolchain mapping toolchain = {'name': 'gompi', 'version': '2018a'} tc_mapping = map_toolchain_hierarchies(toolchain, toolchain, self.modtool) # Update the software version init_config(build_options=build_options) get_toolchain_hierarchy.clear() new_version = '1.x.3' tweaked_spec = map_easyconfig_to_target_tc_hierarchy(ec_spec, tc_mapping, update_build_specs={'version': new_version}, update_dep_versions=False) tweaked_ec = process_easyconfig(tweaked_spec)[0] extensions = tweaked_ec['ec']['exts_list'] # check one extension with the same name exists and that the version has been updated hit_extension = 0 for extension in extensions: if isinstance(extension, tuple) and extension[0] == 'toy': self.assertEqual(extension[1], new_version) # Make sure checksum has been purged self.assertFalse('checksums' in extension[2]) hit_extension += 1 self.assertEqual(hit_extension, 1, "Should only have updated one extension")
def test_find_potential_version_mappings(self): """Test ability to find potential version mappings of a dependency for a given toolchain mapping""" test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') init_config(build_options={ 'robot_path': [test_easyconfigs], 'silent': True, 'valid_module_classes': module_classes(), }) get_toolchain_hierarchy.clear() gcc_binutils_tc = {'name': 'GCC', 'version': '4.9.3-2.26'} iccifort_binutils_tc = {'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'} # The below mapping includes a binutils mapping (2.26 to 2.25) tc_mapping = map_toolchain_hierarchies(gcc_binutils_tc, iccifort_binutils_tc, self.modtool) ec_spec = os.path.join(test_easyconfigs, 'h', 'hwloc', 'hwloc-1.6.2-GCC-4.9.3-2.26.eb') parsed_ec = process_easyconfig(ec_spec)[0] gzip_dep = [dep for dep in parsed_ec['ec']['dependencies'] if dep['name'] == 'gzip'][0] self.assertEqual(gzip_dep['full_mod_name'], 'gzip/1.4-GCC-4.9.3-2.26') potential_versions = find_potential_version_mappings(gzip_dep, tc_mapping) self.assertEqual(len(potential_versions), 1) # Should see version 1.6 of gzip with iccifort toolchain expected = { 'path': os.path.join(test_easyconfigs, 'g', 'gzip', 'gzip-1.6-iccifort-2016.1.150-GCC-4.9.3-2.25.eb'), 'toolchain': {'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'}, 'version': '1.6', 'versionsuffix': '', } self.assertEqual(potential_versions[0], expected) # Test that we can override respecting the versionsuffix # Create toolchain mapping for OpenBLAS gcc_4_tc = {'name': 'GCC', 'version': '4.8.2'} gcc_6_tc = {'name': 'GCC', 'version': '6.4.0-2.28'} tc_mapping = map_toolchain_hierarchies(gcc_4_tc, gcc_6_tc, self.modtool) # Create a dep with the necessary params (including versionsuffix) openblas_dep = { 'toolchain': {'version': '4.8.2', 'name': 'GCC'}, 'name': 'OpenBLAS', 'system': False, 'versionsuffix': '-LAPACK-3.4.2', 'version': '0.2.8' } self.mock_stderr(True) potential_versions = find_potential_version_mappings(openblas_dep, tc_mapping) errtxt = self.get_stderr() warning_stub = "\nWARNING: There may be newer version(s) of dep 'OpenBLAS' available with a different " \ "versionsuffix to '-LAPACK-3.4.2'" self.mock_stderr(False) self.assertTrue(errtxt.startswith(warning_stub)) self.assertEqual(len(potential_versions), 0) potential_versions = find_potential_version_mappings(openblas_dep, tc_mapping, ignore_versionsuffixes=True) self.assertEqual(len(potential_versions), 1) expected = { 'path': os.path.join(test_easyconfigs, 'o', 'OpenBLAS', 'OpenBLAS-0.2.20-GCC-6.4.0-2.28.eb'), 'toolchain': {'version': '6.4.0-2.28', 'name': 'GCC'}, 'version': '0.2.20', 'versionsuffix': '', } self.assertEqual(potential_versions[0], expected)