def find_entries_with_different_content(config_xml, skip_disabled): """ Find entries where their content doesn't match what is published in the information system. """ try: # Find all enabled config entries with ref ids config_dom = minidom.parse(config_xml) config_entries = infosys_lib.parse_entries(config_dom, skip_missing_ref_id=True, skip_disabled=skip_disabled) # Create an info systems list from factory config infosystems = infosys_lib.parse_info_systems(config_dom) has_ress = False for infosys in infosystems: if infosystems[infosys].lower() == 'ress': has_ress = True break if has_ress: # Update path with condor condor_path = infosys_lib.parse_condor_path(config_dom) os.environ["CONDOR_CONFIG"] = condor_path + "/etc/condor_config" condorExe.set_path(condor_path + "/bin", condor_path + "/sbin") except: print("Error parsing the the config file '%s', exiting the tool." % config_xml) sys.exit(2) # Retrieve info systems entries bdii_entries = {} ress_entries = {} tg_entries = {} for infosys, type in infosystems.iteritems(): if type.lower() == 'bdii': bdii_entries.update(infosys_lib.query_bdii(infosys)) elif type.lower() == 'ress': ress_entries.update(infosys_lib.query_ress(infosys)) elif type.lower() == 'tg': tg_entries.update(infosys_lib.query_teragrid()) id_match_bdii_entries = find_entries_id_match(bdii_entries, config_entries, 'bdii') id_match_ress_entries = find_entries_id_match(ress_entries, config_entries, 'ress') id_match_tg_entries = find_entries_id_match(tg_entries, config_entries, 'tg') return id_match_bdii_entries, id_match_ress_entries, id_match_tg_entries
def find_entries_with_ids_not_published(config_xml, skip_disabled): """ Find config entries not published in the information systems. """ try: # Find all enabled config entries with ref ids config_dom = minidom.parse(config_xml) config_entries = infosys_lib.parse_entries(config_dom, skip_missing_ref_id=True, skip_disabled=skip_disabled) # Create an info systems list from factory config infosystems = infosys_lib.parse_info_systems(config_dom) has_ress = False for infosys in infosystems: if infosystems[infosys].lower() == 'ress': has_ress = True break if has_ress: # Update path with condor condor_path = infosys_lib.parse_condor_path(config_dom) os.environ["CONDOR_CONFIG"] = condor_path + "/etc/condor_config" condorExe.set_path(condor_path + "/bin", condor_path + "/sbin") except: print "Error parsing the the config file '%s', exiting the tool." % config_xml sys.exit(2) # Retrieve info systems entries bdii_entries = {} ress_entries = {} tg_entries = {} for infosys, type in infosystems.iteritems(): if type.lower() == 'bdii': bdii_entries.update(infosys_lib.query_bdii(infosys)) elif type.lower() == 'ress': ress_entries.update(infosys_lib.query_ress(infosys)) elif type.lower() == 'tg': tg_entries.update(infosys_lib.query_teragrid()) id_not_found_bdii_entries = find_entries_id_not_found(bdii_entries, config_entries, 'bdii') id_not_found_ress_entries = find_entries_id_not_found(ress_entries, config_entries, 'ress') id_not_found_tg_entries = find_entries_id_not_found(tg_entries, config_entries, 'tg') return id_not_found_bdii_entries, id_not_found_ress_entries, id_not_found_tg_entries
def test_parse_entries(self): """ Verify entries are parsed correctly from the factory configuration file. """ valid_file = open('valid.xml', 'w') valid_file.write( '<glidein schedd_name="[email protected],[email protected]"> \ <entries> \ <entry name="valid_entry" \ enabled="True" gridtype="gt2" verbosity="std" work_dir="OSG" \ gatekeeper="node.fnal.gov/jobmanager-condor" \ rsl="(queue=default)" \ schedd_name="*****@*****.**"> \ <attrs> \ <attr name="CONDOR_ARCH" value="default"/> \ <attr name="GLEXEC_BIN" value="NONE"/> \ <attr name="GLIDEIN_Site" value="VE1"/> \ </attrs> \ <infosys_refs/> \ </entry> \ <entry name="valid_entry2" \ enabled="False" gridtype="cream" verbosity="std" work_dir="." \ gatekeeper="node2.fnal.gov/jobmanager-condor" \ schedd_name="*****@*****.**"> \ <infosys_refs> \ <infosys_ref ref="GlueCEUniqueID=node.fnal.gov:2119/jobmanager-condor_default,Mds-Vo-name=TEST,Mds-Vo-name=local,o=grid" \ server="exp-bdii.cern.ch" type="BDII"/> \ </infosys_refs> \ </entry> \ </entries> \ <condor_tarball arch="default" base_dir="/opt/glidecondor" os="default" \ tar_file="/var/www/html/glidefactory/stage/glidein_v2plus/condor.tgz" version="default"/> \ </glidein>') valid_file.close() config_dom = minidom.parse('valid.xml') # Test valid entries - get all entries = parse_entries(config_dom, skip_missing_ref_id=False, skip_disabled=False) os.remove('valid.xml') self.assertEqual(len(entries), 2) self.assertEqual(entries['valid_entry']['name'], 'valid_entry') self.assertEqual(entries['valid_entry']['enabled'], 'True') self.assertEqual(entries['valid_entry']['gatekeeper'], 'node.fnal.gov/jobmanager-condor') self.assertEqual(entries['valid_entry']['gridtype'], 'gt2') self.assertEqual(entries['valid_entry']['work_dir'], 'OSG') self.assertEqual(entries['valid_entry']['source'], '') self.assertEqual(entries['valid_entry']['source_type'], '') self.assertEqual(entries['valid_entry']['glidein_attrs'], { 'CONDOR_ARCH': 'default', 'GLEXEC_BIN': 'NONE', 'GLIDEIN_Site': 'VE1' }) self.assertEqual(entries['valid_entry2']['name'], 'valid_entry2') self.assertEqual(entries['valid_entry2']['enabled'], 'False') self.assertEqual(entries['valid_entry2']['gatekeeper'], 'node2.fnal.gov/jobmanager-condor') self.assertEqual(entries['valid_entry2']['gridtype'], 'cream') self.assertEqual(entries['valid_entry2']['work_dir'], '.') self.assertEqual(entries['valid_entry2']['source'], 'exp-bdii.cern.ch') self.assertEqual(entries['valid_entry2']['source_type'], 'BDII') self.assertEqual(entries['valid_entry2']['glidein_attrs'], {}) # Test with and without rsl self.assertEqual(entries['valid_entry']['rsl'], '(queue=default)') self.assertEqual(entries['valid_entry2']['rsl'], '') # Test with ref id and without ref id self.assertEqual(entries['valid_entry']['ref_id'], '') self.assertEqual( entries['valid_entry2']['ref_id'], 'GlueCEUniqueID=node.fnal.gov:2119/jobmanager-condor_default,Mds-Vo-name=TEST,Mds-Vo-name=local,o=grid' ) # Test skip missing ref id missing_ref_id_entries = parse_entries(config_dom, skip_missing_ref_id=True, skip_disabled=False) self.assertEqual(len(missing_ref_id_entries), 1) # Test skip disabled disabled_entries = parse_entries(config_dom, skip_missing_ref_id=False, skip_disabled=True) self.assertEqual(len(disabled_entries), 1) # Test skip disabled and missing ref id disabled_entries = parse_entries(config_dom, skip_missing_ref_id=True, skip_disabled=True) self.assertEqual(len(disabled_entries), 0) # Test invalid entry (missing gatekeeper string) invalid_file = open('invalid.xml', 'w') invalid_file.write( '<glidein schedd_name="*****@*****.**"> \ <entries> \ <entry name="valid_entry" \ enabled="True" \ gridtype="gt2" \ schedd_name="*****@*****.**" \ verbosity="std" \ work_dir="."> \ <infosys_refs> \ </infosys_refs> \ </entry> \ </entries> \ <condor_tarball arch="default" base_dir="/opt/glidecondor" os="default" \ tar_file="/var/www/html/glidefactory/stage/glidein_v2plus/condor.tgz" version="default"/> \ </glidein>') invalid_file.close() config_dom = minidom.parse('invalid.xml') self.assertRaises(KeyError, parse_entries, config_dom, skip_missing_ref_id=False, skip_disabled=False) os.remove('invalid.xml') # Test missing entry missing_entry_file = open('missing_entry.xml', 'w') missing_entry_file.write( '<glidein schedd_name="*****@*****.**"> \ <entries> \ </entries> \ <condor_tarball arch="default" base_dir="/opt/glidecondor" os="default" \ tar_file="/var/www/html/glidefactory/stage/glidein_v2plus/condor.tgz" version="default"/> \ </glidein>') missing_entry_file.close() config_dom = minidom.parse('missing_entry.xml') self.assertRaises(KeyError, parse_entries, config_dom, skip_missing_ref_id=False, skip_disabled=False) os.remove('missing_entry.xml') # Test with multiple ref ids (only uses first ref id, for now) with_ref_ids_file = open('with_ref_ids.xml', 'w') with_ref_ids_file.write( '<glidein schedd_name="*****@*****.**"> \ <entries> \ <entry name="valid_entry" \ enabled="True" \ gatekeeper="node.fnal.gov/jobmanager-condor" \ gridtype="gt2" \ schedd_name="*****@*****.**" \ verbosity="std" \ work_dir="."> \ <infosys_refs> \ <infosys_ref ref="GlueCEUniqueID=node.fnal.gov:2119/jobmanager-condor_default,Mds-Vo-name=TEST,Mds-Vo-name=local,o=grid" server="exp-bdii.cern.ch" type="BDII"/> \ <infosys_ref ref="GlueCEUniqueID=node2.fnal.gov:2119/jobmanager-condor_default,Mds-Vo-name=TEST,Mds-Vo-name=local,o=grid" server="exp-bdii.cern.ch" type="BDII"/> \ </infosys_refs> \ </entry> \ </entries> \ <condor_tarball arch="default" base_dir="/opt/glidecondor" os="default" \ tar_file="/var/www/html/glidefactory/stage/glidein_v2plus/condor.tgz" version="default"/> \ </glidein>') with_ref_ids_file.close() config_dom = minidom.parse('with_ref_ids.xml') entries = parse_entries(config_dom) self.assertEqual( entries['valid_entry']['ref_id'], 'GlueCEUniqueID=node.fnal.gov:2119/jobmanager-condor_default,Mds-Vo-name=TEST,Mds-Vo-name=local,o=grid' ) os.remove('with_ref_ids.xml')
def test_parse_entries(self): """ Verify entries are parsed correctly from the factory configuration file. """ valid_file = open('valid.xml', 'w') valid_file.write('<glidein schedd_name="[email protected],[email protected]"> \ <entries> \ <entry name="valid_entry" \ enabled="True" gridtype="gt2" verbosity="std" work_dir="OSG" \ gatekeeper="node.fnal.gov/jobmanager-condor" \ rsl="(queue=default)" \ schedd_name="*****@*****.**"> \ <attrs> \ <attr name="CONDOR_ARCH" value="default"/> \ <attr name="GLEXEC_BIN" value="NONE"/> \ <attr name="GLIDEIN_Site" value="VE1"/> \ </attrs> \ <infosys_refs/> \ </entry> \ <entry name="valid_entry2" \ enabled="False" gridtype="cream" verbosity="std" work_dir="." \ gatekeeper="node2.fnal.gov/jobmanager-condor" \ schedd_name="*****@*****.**"> \ <infosys_refs> \ <infosys_ref ref="GlueCEUniqueID=node.fnal.gov:2119/jobmanager-condor_default,Mds-Vo-name=TEST,Mds-Vo-name=local,o=grid" \ server="exp-bdii.cern.ch" type="BDII"/> \ </infosys_refs> \ </entry> \ </entries> \ <condor_tarball arch="default" base_dir="/opt/glidecondor" os="default" \ tar_file="/var/www/html/glidefactory/stage/glidein_v2plus/condor.tgz" version="default"/> \ </glidein>') valid_file.close() config_dom = minidom.parse('valid.xml') # Test valid entries - get all entries = parse_entries(config_dom, skip_missing_ref_id=False, skip_disabled=False) os.remove('valid.xml') self.assertEqual(len(entries), 2) self.assertEqual(entries['valid_entry']['name'], 'valid_entry') self.assertEqual(entries['valid_entry']['enabled'], 'True') self.assertEqual(entries['valid_entry']['gatekeeper'], 'node.fnal.gov/jobmanager-condor') self.assertEqual(entries['valid_entry']['gridtype'], 'gt2') self.assertEqual(entries['valid_entry']['work_dir'], 'OSG') self.assertEqual(entries['valid_entry']['source'], '') self.assertEqual(entries['valid_entry']['source_type'], '') self.assertEqual(entries['valid_entry']['glidein_attrs'], {'CONDOR_ARCH':'default', 'GLEXEC_BIN':'NONE', 'GLIDEIN_Site':'VE1'}) self.assertEqual(entries['valid_entry2']['name'], 'valid_entry2') self.assertEqual(entries['valid_entry2']['enabled'], 'False') self.assertEqual(entries['valid_entry2']['gatekeeper'], 'node2.fnal.gov/jobmanager-condor') self.assertEqual(entries['valid_entry2']['gridtype'], 'cream') self.assertEqual(entries['valid_entry2']['work_dir'], '.') self.assertEqual(entries['valid_entry2']['source'], 'exp-bdii.cern.ch') self.assertEqual(entries['valid_entry2']['source_type'], 'BDII') self.assertEqual(entries['valid_entry2']['glidein_attrs'], {}) # Test with and without rsl self.assertEqual(entries['valid_entry']['rsl'], '(queue=default)') self.assertEqual(entries['valid_entry2']['rsl'], '') # Test with ref id and without ref id self.assertEqual(entries['valid_entry']['ref_id'], '') self.assertEqual(entries['valid_entry2']['ref_id'], 'GlueCEUniqueID=node.fnal.gov:2119/jobmanager-condor_default,Mds-Vo-name=TEST,Mds-Vo-name=local,o=grid') # Test skip missing ref id missing_ref_id_entries = parse_entries(config_dom, skip_missing_ref_id=True, skip_disabled=False) self.assertEqual(len(missing_ref_id_entries), 1) # Test skip disabled disabled_entries = parse_entries(config_dom, skip_missing_ref_id=False, skip_disabled=True) self.assertEqual(len(disabled_entries), 1) # Test skip disabled and missing ref id disabled_entries = parse_entries(config_dom, skip_missing_ref_id=True, skip_disabled=True) self.assertEqual(len(disabled_entries), 0) # Test invalid entry (missing gatekeeper string) invalid_file = open('invalid.xml', 'w') invalid_file.write('<glidein schedd_name="*****@*****.**"> \ <entries> \ <entry name="valid_entry" \ enabled="True" \ gridtype="gt2" \ schedd_name="*****@*****.**" \ verbosity="std" \ work_dir="."> \ <infosys_refs> \ </infosys_refs> \ </entry> \ </entries> \ <condor_tarball arch="default" base_dir="/opt/glidecondor" os="default" \ tar_file="/var/www/html/glidefactory/stage/glidein_v2plus/condor.tgz" version="default"/> \ </glidein>') invalid_file.close() config_dom = minidom.parse('invalid.xml') self.assertRaises(KeyError, parse_entries, config_dom, skip_missing_ref_id=False, skip_disabled=False) os.remove('invalid.xml') # Test missing entry missing_entry_file = open('missing_entry.xml', 'w') missing_entry_file.write('<glidein schedd_name="*****@*****.**"> \ <entries> \ </entries> \ <condor_tarball arch="default" base_dir="/opt/glidecondor" os="default" \ tar_file="/var/www/html/glidefactory/stage/glidein_v2plus/condor.tgz" version="default"/> \ </glidein>') missing_entry_file.close() config_dom = minidom.parse('missing_entry.xml') self.assertRaises(KeyError, parse_entries, config_dom, skip_missing_ref_id=False, skip_disabled=False) os.remove('missing_entry.xml') # Test with multiple ref ids (only uses first ref id, for now) with_ref_ids_file = open('with_ref_ids.xml', 'w') with_ref_ids_file.write('<glidein schedd_name="*****@*****.**"> \ <entries> \ <entry name="valid_entry" \ enabled="True" \ gatekeeper="node.fnal.gov/jobmanager-condor" \ gridtype="gt2" \ schedd_name="*****@*****.**" \ verbosity="std" \ work_dir="."> \ <infosys_refs> \ <infosys_ref ref="GlueCEUniqueID=node.fnal.gov:2119/jobmanager-condor_default,Mds-Vo-name=TEST,Mds-Vo-name=local,o=grid" server="exp-bdii.cern.ch" type="BDII"/> \ <infosys_ref ref="GlueCEUniqueID=node2.fnal.gov:2119/jobmanager-condor_default,Mds-Vo-name=TEST,Mds-Vo-name=local,o=grid" server="exp-bdii.cern.ch" type="BDII"/> \ </infosys_refs> \ </entry> \ </entries> \ <condor_tarball arch="default" base_dir="/opt/glidecondor" os="default" \ tar_file="/var/www/html/glidefactory/stage/glidein_v2plus/condor.tgz" version="default"/> \ </glidein>') with_ref_ids_file.close() config_dom = minidom.parse('with_ref_ids.xml') entries = parse_entries(config_dom) self.assertEqual(entries['valid_entry']['ref_id'], 'GlueCEUniqueID=node.fnal.gov:2119/jobmanager-condor_default,Mds-Vo-name=TEST,Mds-Vo-name=local,o=grid') os.remove('with_ref_ids.xml')