def find_entries_with_different_content(config_xml, skip_disabled):
    """
    Find entries where their content doesn't match what is published in the information system. 
    """
    try:
        # Find all enabled config entries with ref ids
        config_dom = minidom.parse(config_xml)
        config_entries = infosys_lib.parse_entries(config_dom,
                                                   skip_missing_ref_id=True,
                                                   skip_disabled=skip_disabled)

        # Create an info systems list from factory config
        infosystems = infosys_lib.parse_info_systems(config_dom)

        has_ress = False
        for infosys in infosystems:
            if infosystems[infosys].lower() == 'ress':
                has_ress = True
                break
        if has_ress:
            # Update path with condor
            condor_path = infosys_lib.parse_condor_path(config_dom)
            os.environ["CONDOR_CONFIG"] = condor_path + "/etc/condor_config"
            condorExe.set_path(condor_path + "/bin", condor_path + "/sbin")
    except:
        print("Error parsing the the config file '%s', exiting the tool." %
              config_xml)
        sys.exit(2)

    # Retrieve info systems entries
    bdii_entries = {}
    ress_entries = {}
    tg_entries = {}
    for infosys, type in infosystems.iteritems():
        if type.lower() == 'bdii':
            bdii_entries.update(infosys_lib.query_bdii(infosys))

        elif type.lower() == 'ress':
            ress_entries.update(infosys_lib.query_ress(infosys))

        elif type.lower() == 'tg':
            tg_entries.update(infosys_lib.query_teragrid())

    id_match_bdii_entries = find_entries_id_match(bdii_entries, config_entries,
                                                  'bdii')
    id_match_ress_entries = find_entries_id_match(ress_entries, config_entries,
                                                  'ress')
    id_match_tg_entries = find_entries_id_match(tg_entries, config_entries,
                                                'tg')

    return id_match_bdii_entries, id_match_ress_entries, id_match_tg_entries
def find_entries_with_ids_not_published(config_xml, skip_disabled):
    """
    Find config entries not published in the information systems.
    """
    try:
        # Find all enabled config entries with ref ids
        config_dom = minidom.parse(config_xml)
        config_entries = infosys_lib.parse_entries(config_dom, skip_missing_ref_id=True, skip_disabled=skip_disabled)
                    
        # Create an info systems list from factory config
        infosystems = infosys_lib.parse_info_systems(config_dom)
        
        has_ress = False
        for infosys in infosystems:
            if infosystems[infosys].lower() == 'ress':
                has_ress = True
                break
        if has_ress:  
            # Update path with condor 
            condor_path = infosys_lib.parse_condor_path(config_dom)
            os.environ["CONDOR_CONFIG"] = condor_path + "/etc/condor_config"
            condorExe.set_path(condor_path + "/bin", condor_path + "/sbin")
    except: 
        print "Error parsing the the config file '%s', exiting the tool." % config_xml
        sys.exit(2) 

    # Retrieve info systems entries 
    bdii_entries = {}
    ress_entries = {}
    tg_entries = {}
    for infosys, type in infosystems.iteritems():
        if type.lower() == 'bdii':
            bdii_entries.update(infosys_lib.query_bdii(infosys))
                
        elif type.lower() == 'ress':
            ress_entries.update(infosys_lib.query_ress(infosys))
                
        elif type.lower() == 'tg':
            tg_entries.update(infosys_lib.query_teragrid())
            

    id_not_found_bdii_entries = find_entries_id_not_found(bdii_entries, config_entries, 'bdii')
    id_not_found_ress_entries = find_entries_id_not_found(ress_entries, config_entries, 'ress')
    id_not_found_tg_entries = find_entries_id_not_found(tg_entries, config_entries, 'tg')

    return id_not_found_bdii_entries, id_not_found_ress_entries, id_not_found_tg_entries
    def test_parse_entries(self):
        """
        Verify entries are parsed correctly from the factory configuration file.
        """

        valid_file = open('valid.xml', 'w')
        valid_file.write(
            '<glidein schedd_name="[email protected],[email protected]">  \
                <entries> \
                    <entry name="valid_entry"  \
                            enabled="True"  gridtype="gt2" verbosity="std" work_dir="OSG"  \
                            gatekeeper="node.fnal.gov/jobmanager-condor"  \
                            rsl="(queue=default)"  \
                            schedd_name="*****@*****.**">  \
                        <attrs>  \
                            <attr name="CONDOR_ARCH" value="default"/>  \
                            <attr name="GLEXEC_BIN" value="NONE"/>  \
                            <attr name="GLIDEIN_Site" value="VE1"/>  \
                        </attrs>  \
                        <infosys_refs/>  \
                    </entry>  \
                    <entry name="valid_entry2"  \
                            enabled="False" gridtype="cream" verbosity="std" work_dir="."  \
                            gatekeeper="node2.fnal.gov/jobmanager-condor"  \
                            schedd_name="*****@*****.**">  \
                        <infosys_refs>  \
                            <infosys_ref ref="GlueCEUniqueID=node.fnal.gov:2119/jobmanager-condor_default,Mds-Vo-name=TEST,Mds-Vo-name=local,o=grid"  \
                            server="exp-bdii.cern.ch" type="BDII"/>  \
                        </infosys_refs>  \
                    </entry>  \
                </entries>  \
                <condor_tarball arch="default" base_dir="/opt/glidecondor" os="default"  \
                            tar_file="/var/www/html/glidefactory/stage/glidein_v2plus/condor.tgz" version="default"/>  \
            </glidein>')
        valid_file.close()
        config_dom = minidom.parse('valid.xml')

        # Test valid entries - get all
        entries = parse_entries(config_dom,
                                skip_missing_ref_id=False,
                                skip_disabled=False)
        os.remove('valid.xml')
        self.assertEqual(len(entries), 2)
        self.assertEqual(entries['valid_entry']['name'], 'valid_entry')
        self.assertEqual(entries['valid_entry']['enabled'], 'True')
        self.assertEqual(entries['valid_entry']['gatekeeper'],
                         'node.fnal.gov/jobmanager-condor')
        self.assertEqual(entries['valid_entry']['gridtype'], 'gt2')
        self.assertEqual(entries['valid_entry']['work_dir'], 'OSG')
        self.assertEqual(entries['valid_entry']['source'], '')
        self.assertEqual(entries['valid_entry']['source_type'], '')
        self.assertEqual(entries['valid_entry']['glidein_attrs'], {
            'CONDOR_ARCH': 'default',
            'GLEXEC_BIN': 'NONE',
            'GLIDEIN_Site': 'VE1'
        })

        self.assertEqual(entries['valid_entry2']['name'], 'valid_entry2')
        self.assertEqual(entries['valid_entry2']['enabled'], 'False')
        self.assertEqual(entries['valid_entry2']['gatekeeper'],
                         'node2.fnal.gov/jobmanager-condor')
        self.assertEqual(entries['valid_entry2']['gridtype'], 'cream')
        self.assertEqual(entries['valid_entry2']['work_dir'], '.')
        self.assertEqual(entries['valid_entry2']['source'], 'exp-bdii.cern.ch')
        self.assertEqual(entries['valid_entry2']['source_type'], 'BDII')
        self.assertEqual(entries['valid_entry2']['glidein_attrs'], {})

        # Test with and without rsl
        self.assertEqual(entries['valid_entry']['rsl'], '(queue=default)')
        self.assertEqual(entries['valid_entry2']['rsl'], '')

        # Test with ref id and without ref id
        self.assertEqual(entries['valid_entry']['ref_id'], '')
        self.assertEqual(
            entries['valid_entry2']['ref_id'],
            'GlueCEUniqueID=node.fnal.gov:2119/jobmanager-condor_default,Mds-Vo-name=TEST,Mds-Vo-name=local,o=grid'
        )

        # Test skip missing ref id
        missing_ref_id_entries = parse_entries(config_dom,
                                               skip_missing_ref_id=True,
                                               skip_disabled=False)
        self.assertEqual(len(missing_ref_id_entries), 1)

        # Test skip disabled
        disabled_entries = parse_entries(config_dom,
                                         skip_missing_ref_id=False,
                                         skip_disabled=True)
        self.assertEqual(len(disabled_entries), 1)

        # Test skip disabled and missing ref id
        disabled_entries = parse_entries(config_dom,
                                         skip_missing_ref_id=True,
                                         skip_disabled=True)
        self.assertEqual(len(disabled_entries), 0)

        # Test invalid entry (missing gatekeeper string)
        invalid_file = open('invalid.xml', 'w')
        invalid_file.write(
            '<glidein schedd_name="*****@*****.**">  \
                <entries>  \
                    <entry name="valid_entry"  \
                            enabled="True"  \
                            gridtype="gt2"  \
                            schedd_name="*****@*****.**"  \
                            verbosity="std"  \
                            work_dir=".">  \
                        <infosys_refs>  \
                        </infosys_refs>  \
                    </entry>  \
                </entries>  \
                <condor_tarball arch="default" base_dir="/opt/glidecondor" os="default"   \
                            tar_file="/var/www/html/glidefactory/stage/glidein_v2plus/condor.tgz" version="default"/>  \
            </glidein>')
        invalid_file.close()
        config_dom = minidom.parse('invalid.xml')
        self.assertRaises(KeyError,
                          parse_entries,
                          config_dom,
                          skip_missing_ref_id=False,
                          skip_disabled=False)
        os.remove('invalid.xml')

        # Test missing entry
        missing_entry_file = open('missing_entry.xml', 'w')
        missing_entry_file.write(
            '<glidein schedd_name="*****@*****.**">  \
                <entries>  \
                </entries>  \
                <condor_tarball arch="default" base_dir="/opt/glidecondor" os="default"   \
                            tar_file="/var/www/html/glidefactory/stage/glidein_v2plus/condor.tgz" version="default"/>  \
            </glidein>')
        missing_entry_file.close()
        config_dom = minidom.parse('missing_entry.xml')
        self.assertRaises(KeyError,
                          parse_entries,
                          config_dom,
                          skip_missing_ref_id=False,
                          skip_disabled=False)
        os.remove('missing_entry.xml')

        # Test with multiple ref ids (only uses first ref id, for now)
        with_ref_ids_file = open('with_ref_ids.xml', 'w')
        with_ref_ids_file.write(
            '<glidein schedd_name="*****@*****.**">  \
                <entries> \
                    <entry name="valid_entry"  \
                            enabled="True"  \
                            gatekeeper="node.fnal.gov/jobmanager-condor"  \
                            gridtype="gt2"  \
                            schedd_name="*****@*****.**"  \
                            verbosity="std"  \
                            work_dir=".">  \
                        <infosys_refs>  \
                            <infosys_ref ref="GlueCEUniqueID=node.fnal.gov:2119/jobmanager-condor_default,Mds-Vo-name=TEST,Mds-Vo-name=local,o=grid" server="exp-bdii.cern.ch" type="BDII"/>  \
                            <infosys_ref ref="GlueCEUniqueID=node2.fnal.gov:2119/jobmanager-condor_default,Mds-Vo-name=TEST,Mds-Vo-name=local,o=grid" server="exp-bdii.cern.ch" type="BDII"/>  \
                        </infosys_refs>  \
                    </entry>  \
                </entries>  \
                <condor_tarball arch="default" base_dir="/opt/glidecondor" os="default"  \
                            tar_file="/var/www/html/glidefactory/stage/glidein_v2plus/condor.tgz" version="default"/>  \
            </glidein>')
        with_ref_ids_file.close()
        config_dom = minidom.parse('with_ref_ids.xml')
        entries = parse_entries(config_dom)
        self.assertEqual(
            entries['valid_entry']['ref_id'],
            'GlueCEUniqueID=node.fnal.gov:2119/jobmanager-condor_default,Mds-Vo-name=TEST,Mds-Vo-name=local,o=grid'
        )
        os.remove('with_ref_ids.xml')
 def test_parse_entries(self):
     """
     Verify entries are parsed correctly from the factory configuration file.
     """
     
     valid_file = open('valid.xml', 'w')
     valid_file.write('<glidein schedd_name="[email protected],[email protected]">  \
             <entries> \
                 <entry name="valid_entry"  \
                         enabled="True"  gridtype="gt2" verbosity="std" work_dir="OSG"  \
                         gatekeeper="node.fnal.gov/jobmanager-condor"  \
                         rsl="(queue=default)"  \
                         schedd_name="*****@*****.**">  \
                     <attrs>  \
                         <attr name="CONDOR_ARCH" value="default"/>  \
                         <attr name="GLEXEC_BIN" value="NONE"/>  \
                         <attr name="GLIDEIN_Site" value="VE1"/>  \
                     </attrs>  \
                     <infosys_refs/>  \
                 </entry>  \
                 <entry name="valid_entry2"  \
                         enabled="False" gridtype="cream" verbosity="std" work_dir="."  \
                         gatekeeper="node2.fnal.gov/jobmanager-condor"  \
                         schedd_name="*****@*****.**">  \
                     <infosys_refs>  \
                         <infosys_ref ref="GlueCEUniqueID=node.fnal.gov:2119/jobmanager-condor_default,Mds-Vo-name=TEST,Mds-Vo-name=local,o=grid"  \
                         server="exp-bdii.cern.ch" type="BDII"/>  \
                     </infosys_refs>  \
                 </entry>  \
             </entries>  \
             <condor_tarball arch="default" base_dir="/opt/glidecondor" os="default"  \
                         tar_file="/var/www/html/glidefactory/stage/glidein_v2plus/condor.tgz" version="default"/>  \
         </glidein>')
     valid_file.close()
     config_dom = minidom.parse('valid.xml')
     
     # Test valid entries - get all
     entries = parse_entries(config_dom, skip_missing_ref_id=False, skip_disabled=False)
     os.remove('valid.xml')
     self.assertEqual(len(entries), 2)
     self.assertEqual(entries['valid_entry']['name'], 'valid_entry')
     self.assertEqual(entries['valid_entry']['enabled'], 'True')
     self.assertEqual(entries['valid_entry']['gatekeeper'], 'node.fnal.gov/jobmanager-condor')
     self.assertEqual(entries['valid_entry']['gridtype'], 'gt2')
     self.assertEqual(entries['valid_entry']['work_dir'], 'OSG')
     self.assertEqual(entries['valid_entry']['source'], '')
     self.assertEqual(entries['valid_entry']['source_type'], '')
     self.assertEqual(entries['valid_entry']['glidein_attrs'], {'CONDOR_ARCH':'default', 'GLEXEC_BIN':'NONE', 'GLIDEIN_Site':'VE1'})
     
     self.assertEqual(entries['valid_entry2']['name'], 'valid_entry2')
     self.assertEqual(entries['valid_entry2']['enabled'], 'False')
     self.assertEqual(entries['valid_entry2']['gatekeeper'], 'node2.fnal.gov/jobmanager-condor')
     self.assertEqual(entries['valid_entry2']['gridtype'], 'cream')
     self.assertEqual(entries['valid_entry2']['work_dir'], '.')
     self.assertEqual(entries['valid_entry2']['source'], 'exp-bdii.cern.ch')
     self.assertEqual(entries['valid_entry2']['source_type'], 'BDII')
     self.assertEqual(entries['valid_entry2']['glidein_attrs'], {})
     
     # Test with and without rsl
     self.assertEqual(entries['valid_entry']['rsl'], '(queue=default)')
     self.assertEqual(entries['valid_entry2']['rsl'], '')        
     
     # Test with ref id and without ref id
     self.assertEqual(entries['valid_entry']['ref_id'], '')
     self.assertEqual(entries['valid_entry2']['ref_id'], 'GlueCEUniqueID=node.fnal.gov:2119/jobmanager-condor_default,Mds-Vo-name=TEST,Mds-Vo-name=local,o=grid')        
     
     # Test skip missing ref id
     missing_ref_id_entries = parse_entries(config_dom, skip_missing_ref_id=True, skip_disabled=False)
     self.assertEqual(len(missing_ref_id_entries), 1)
     
     # Test skip disabled
     disabled_entries = parse_entries(config_dom, skip_missing_ref_id=False, skip_disabled=True)
     self.assertEqual(len(disabled_entries), 1)    
     
     # Test skip disabled and missing ref id
     disabled_entries = parse_entries(config_dom, skip_missing_ref_id=True, skip_disabled=True)
     self.assertEqual(len(disabled_entries), 0)        
     
     # Test invalid entry (missing gatekeeper string)
     invalid_file = open('invalid.xml', 'w')
     invalid_file.write('<glidein schedd_name="*****@*****.**">  \
             <entries>  \
                 <entry name="valid_entry"  \
                         enabled="True"  \
                         gridtype="gt2"  \
                         schedd_name="*****@*****.**"  \
                         verbosity="std"  \
                         work_dir=".">  \
                     <infosys_refs>  \
                     </infosys_refs>  \
                 </entry>  \
             </entries>  \
             <condor_tarball arch="default" base_dir="/opt/glidecondor" os="default"   \
                         tar_file="/var/www/html/glidefactory/stage/glidein_v2plus/condor.tgz" version="default"/>  \
         </glidein>')
     invalid_file.close()
     config_dom = minidom.parse('invalid.xml')
     self.assertRaises(KeyError, parse_entries, config_dom, skip_missing_ref_id=False, skip_disabled=False)
     os.remove('invalid.xml')
     
     # Test missing entry  
     missing_entry_file = open('missing_entry.xml', 'w')
     missing_entry_file.write('<glidein schedd_name="*****@*****.**">  \
             <entries>  \
             </entries>  \
             <condor_tarball arch="default" base_dir="/opt/glidecondor" os="default"   \
                         tar_file="/var/www/html/glidefactory/stage/glidein_v2plus/condor.tgz" version="default"/>  \
         </glidein>')
     missing_entry_file.close()
     config_dom = minidom.parse('missing_entry.xml')
     self.assertRaises(KeyError, parse_entries, config_dom, skip_missing_ref_id=False, skip_disabled=False)
     os.remove('missing_entry.xml')     
                 
     # Test with multiple ref ids (only uses first ref id, for now)
     with_ref_ids_file = open('with_ref_ids.xml', 'w')
     with_ref_ids_file.write('<glidein schedd_name="*****@*****.**">  \
             <entries> \
                 <entry name="valid_entry"  \
                         enabled="True"  \
                         gatekeeper="node.fnal.gov/jobmanager-condor"  \
                         gridtype="gt2"  \
                         schedd_name="*****@*****.**"  \
                         verbosity="std"  \
                         work_dir=".">  \
                     <infosys_refs>  \
                         <infosys_ref ref="GlueCEUniqueID=node.fnal.gov:2119/jobmanager-condor_default,Mds-Vo-name=TEST,Mds-Vo-name=local,o=grid" server="exp-bdii.cern.ch" type="BDII"/>  \
                         <infosys_ref ref="GlueCEUniqueID=node2.fnal.gov:2119/jobmanager-condor_default,Mds-Vo-name=TEST,Mds-Vo-name=local,o=grid" server="exp-bdii.cern.ch" type="BDII"/>  \
                     </infosys_refs>  \
                 </entry>  \
             </entries>  \
             <condor_tarball arch="default" base_dir="/opt/glidecondor" os="default"  \
                         tar_file="/var/www/html/glidefactory/stage/glidein_v2plus/condor.tgz" version="default"/>  \
         </glidein>')
     with_ref_ids_file.close()
     config_dom = minidom.parse('with_ref_ids.xml')
     entries = parse_entries(config_dom)
     self.assertEqual(entries['valid_entry']['ref_id'], 'GlueCEUniqueID=node.fnal.gov:2119/jobmanager-condor_default,Mds-Vo-name=TEST,Mds-Vo-name=local,o=grid')
     os.remove('with_ref_ids.xml')