def find_entries_with_different_content(config_xml, skip_disabled):
"""
Find entries where their content doesn't match what is published in the information system.
"""
try:
# Find all enabled config entries with ref ids
config_dom = minidom.parse(config_xml)
config_entries = infosys_lib.parse_entries(config_dom,
skip_missing_ref_id=True,
skip_disabled=skip_disabled)
# Create an info systems list from factory config
infosystems = infosys_lib.parse_info_systems(config_dom)
has_ress = False
for infosys in infosystems:
if infosystems[infosys].lower() == 'ress':
has_ress = True
break
if has_ress:
# Update path with condor
condor_path = infosys_lib.parse_condor_path(config_dom)
os.environ["CONDOR_CONFIG"] = condor_path + "/etc/condor_config"
condorExe.set_path(condor_path + "/bin", condor_path + "/sbin")
except:
print("Error parsing the the config file '%s', exiting the tool." %
config_xml)
sys.exit(2)
# Retrieve info systems entries
bdii_entries = {}
ress_entries = {}
tg_entries = {}
for infosys, type in infosystems.iteritems():
if type.lower() == 'bdii':
bdii_entries.update(infosys_lib.query_bdii(infosys))
elif type.lower() == 'ress':
ress_entries.update(infosys_lib.query_ress(infosys))
elif type.lower() == 'tg':
tg_entries.update(infosys_lib.query_teragrid())
id_match_bdii_entries = find_entries_id_match(bdii_entries, config_entries,
'bdii')
id_match_ress_entries = find_entries_id_match(ress_entries, config_entries,
'ress')
id_match_tg_entries = find_entries_id_match(tg_entries, config_entries,
'tg')
return id_match_bdii_entries, id_match_ress_entries, id_match_tg_entries
def find_entries_with_ids_not_published(config_xml, skip_disabled):
"""
Find config entries not published in the information systems.
"""
try:
# Find all enabled config entries with ref ids
config_dom = minidom.parse(config_xml)
config_entries = infosys_lib.parse_entries(config_dom, skip_missing_ref_id=True, skip_disabled=skip_disabled)
# Create an info systems list from factory config
infosystems = infosys_lib.parse_info_systems(config_dom)
has_ress = False
for infosys in infosystems:
if infosystems[infosys].lower() == 'ress':
has_ress = True
break
if has_ress:
# Update path with condor
condor_path = infosys_lib.parse_condor_path(config_dom)
os.environ["CONDOR_CONFIG"] = condor_path + "/etc/condor_config"
condorExe.set_path(condor_path + "/bin", condor_path + "/sbin")
except:
print "Error parsing the the config file '%s', exiting the tool." % config_xml
sys.exit(2)
# Retrieve info systems entries
bdii_entries = {}
ress_entries = {}
tg_entries = {}
for infosys, type in infosystems.iteritems():
if type.lower() == 'bdii':
bdii_entries.update(infosys_lib.query_bdii(infosys))
elif type.lower() == 'ress':
ress_entries.update(infosys_lib.query_ress(infosys))
elif type.lower() == 'tg':
tg_entries.update(infosys_lib.query_teragrid())
id_not_found_bdii_entries = find_entries_id_not_found(bdii_entries, config_entries, 'bdii')
id_not_found_ress_entries = find_entries_id_not_found(ress_entries, config_entries, 'ress')
id_not_found_tg_entries = find_entries_id_not_found(tg_entries, config_entries, 'tg')
return id_not_found_bdii_entries, id_not_found_ress_entries, id_not_found_tg_entries
def test_parse_entries(self):
"""
Verify entries are parsed correctly from the factory configuration file.
"""
valid_file = open('valid.xml', 'w')
valid_file.write(
'<glidein schedd_name="[email protected],[email protected]"> \
<entries> \
<entry name="valid_entry" \
enabled="True" gridtype="gt2" verbosity="std" work_dir="OSG" \
gatekeeper="node.fnal.gov/jobmanager-condor" \
rsl="(queue=default)" \
schedd_name="*****@*****.**"> \
<attrs> \
<attr name="CONDOR_ARCH" value="default"/> \
<attr name="GLEXEC_BIN" value="NONE"/> \
<attr name="GLIDEIN_Site" value="VE1"/> \
</attrs> \
<infosys_refs/> \
</entry> \
<entry name="valid_entry2" \
enabled="False" gridtype="cream" verbosity="std" work_dir="." \
gatekeeper="node2.fnal.gov/jobmanager-condor" \
schedd_name="*****@*****.**"> \
<infosys_refs> \
<infosys_ref ref="GlueCEUniqueID=node.fnal.gov:2119/jobmanager-condor_default,Mds-Vo-name=TEST,Mds-Vo-name=local,o=grid" \
server="exp-bdii.cern.ch" type="BDII"/> \
</infosys_refs> \
</entry> \
</entries> \
<condor_tarball arch="default" base_dir="/opt/glidecondor" os="default" \
tar_file="/var/www/html/glidefactory/stage/glidein_v2plus/condor.tgz" version="default"/> \
</glidein>')
valid_file.close()
config_dom = minidom.parse('valid.xml')
# Test valid entries - get all
entries = parse_entries(config_dom,
skip_missing_ref_id=False,
skip_disabled=False)
os.remove('valid.xml')
self.assertEqual(len(entries), 2)
self.assertEqual(entries['valid_entry']['name'], 'valid_entry')
self.assertEqual(entries['valid_entry']['enabled'], 'True')
self.assertEqual(entries['valid_entry']['gatekeeper'],
'node.fnal.gov/jobmanager-condor')
self.assertEqual(entries['valid_entry']['gridtype'], 'gt2')
self.assertEqual(entries['valid_entry']['work_dir'], 'OSG')
self.assertEqual(entries['valid_entry']['source'], '')
self.assertEqual(entries['valid_entry']['source_type'], '')
self.assertEqual(entries['valid_entry']['glidein_attrs'], {
'CONDOR_ARCH': 'default',
'GLEXEC_BIN': 'NONE',
'GLIDEIN_Site': 'VE1'
})
self.assertEqual(entries['valid_entry2']['name'], 'valid_entry2')
self.assertEqual(entries['valid_entry2']['enabled'], 'False')
self.assertEqual(entries['valid_entry2']['gatekeeper'],
'node2.fnal.gov/jobmanager-condor')
self.assertEqual(entries['valid_entry2']['gridtype'], 'cream')
self.assertEqual(entries['valid_entry2']['work_dir'], '.')
self.assertEqual(entries['valid_entry2']['source'], 'exp-bdii.cern.ch')
self.assertEqual(entries['valid_entry2']['source_type'], 'BDII')
self.assertEqual(entries['valid_entry2']['glidein_attrs'], {})
# Test with and without rsl
self.assertEqual(entries['valid_entry']['rsl'], '(queue=default)')
self.assertEqual(entries['valid_entry2']['rsl'], '')
# Test with ref id and without ref id
self.assertEqual(entries['valid_entry']['ref_id'], '')
self.assertEqual(
entries['valid_entry2']['ref_id'],
'GlueCEUniqueID=node.fnal.gov:2119/jobmanager-condor_default,Mds-Vo-name=TEST,Mds-Vo-name=local,o=grid'
)
# Test skip missing ref id
missing_ref_id_entries = parse_entries(config_dom,
skip_missing_ref_id=True,
skip_disabled=False)
self.assertEqual(len(missing_ref_id_entries), 1)
# Test skip disabled
disabled_entries = parse_entries(config_dom,
skip_missing_ref_id=False,
skip_disabled=True)
self.assertEqual(len(disabled_entries), 1)
# Test skip disabled and missing ref id
disabled_entries = parse_entries(config_dom,
skip_missing_ref_id=True,
skip_disabled=True)
self.assertEqual(len(disabled_entries), 0)
# Test invalid entry (missing gatekeeper string)
invalid_file = open('invalid.xml', 'w')
invalid_file.write(
'<glidein schedd_name="*****@*****.**"> \
<entries> \
<entry name="valid_entry" \
enabled="True" \
gridtype="gt2" \
schedd_name="*****@*****.**" \
verbosity="std" \
work_dir="."> \
<infosys_refs> \
</infosys_refs> \
</entry> \
</entries> \
<condor_tarball arch="default" base_dir="/opt/glidecondor" os="default" \
tar_file="/var/www/html/glidefactory/stage/glidein_v2plus/condor.tgz" version="default"/> \
</glidein>')
invalid_file.close()
config_dom = minidom.parse('invalid.xml')
self.assertRaises(KeyError,
parse_entries,
config_dom,
skip_missing_ref_id=False,
skip_disabled=False)
os.remove('invalid.xml')
# Test missing entry
missing_entry_file = open('missing_entry.xml', 'w')
missing_entry_file.write(
'<glidein schedd_name="*****@*****.**"> \
<entries> \
</entries> \
<condor_tarball arch="default" base_dir="/opt/glidecondor" os="default" \
tar_file="/var/www/html/glidefactory/stage/glidein_v2plus/condor.tgz" version="default"/> \
</glidein>')
missing_entry_file.close()
config_dom = minidom.parse('missing_entry.xml')
self.assertRaises(KeyError,
parse_entries,
config_dom,
skip_missing_ref_id=False,
skip_disabled=False)
os.remove('missing_entry.xml')
# Test with multiple ref ids (only uses first ref id, for now)
with_ref_ids_file = open('with_ref_ids.xml', 'w')
with_ref_ids_file.write(
'<glidein schedd_name="*****@*****.**"> \
<entries> \
<entry name="valid_entry" \
enabled="True" \
gatekeeper="node.fnal.gov/jobmanager-condor" \
gridtype="gt2" \
schedd_name="*****@*****.**" \
verbosity="std" \
work_dir="."> \
<infosys_refs> \
<infosys_ref ref="GlueCEUniqueID=node.fnal.gov:2119/jobmanager-condor_default,Mds-Vo-name=TEST,Mds-Vo-name=local,o=grid" server="exp-bdii.cern.ch" type="BDII"/> \
<infosys_ref ref="GlueCEUniqueID=node2.fnal.gov:2119/jobmanager-condor_default,Mds-Vo-name=TEST,Mds-Vo-name=local,o=grid" server="exp-bdii.cern.ch" type="BDII"/> \
</infosys_refs> \
</entry> \
</entries> \
<condor_tarball arch="default" base_dir="/opt/glidecondor" os="default" \
tar_file="/var/www/html/glidefactory/stage/glidein_v2plus/condor.tgz" version="default"/> \
</glidein>')
with_ref_ids_file.close()
config_dom = minidom.parse('with_ref_ids.xml')
entries = parse_entries(config_dom)
self.assertEqual(
entries['valid_entry']['ref_id'],
'GlueCEUniqueID=node.fnal.gov:2119/jobmanager-condor_default,Mds-Vo-name=TEST,Mds-Vo-name=local,o=grid'
)
os.remove('with_ref_ids.xml')
def test_parse_entries(self):
"""
Verify entries are parsed correctly from the factory configuration file.
"""
valid_file = open('valid.xml', 'w')
valid_file.write('<glidein schedd_name="[email protected],[email protected]"> \
<entries> \
<entry name="valid_entry" \
enabled="True" gridtype="gt2" verbosity="std" work_dir="OSG" \
gatekeeper="node.fnal.gov/jobmanager-condor" \
rsl="(queue=default)" \
schedd_name="*****@*****.**"> \
<attrs> \
<attr name="CONDOR_ARCH" value="default"/> \
<attr name="GLEXEC_BIN" value="NONE"/> \
<attr name="GLIDEIN_Site" value="VE1"/> \
</attrs> \
<infosys_refs/> \
</entry> \
<entry name="valid_entry2" \
enabled="False" gridtype="cream" verbosity="std" work_dir="." \
gatekeeper="node2.fnal.gov/jobmanager-condor" \
schedd_name="*****@*****.**"> \
<infosys_refs> \
<infosys_ref ref="GlueCEUniqueID=node.fnal.gov:2119/jobmanager-condor_default,Mds-Vo-name=TEST,Mds-Vo-name=local,o=grid" \
server="exp-bdii.cern.ch" type="BDII"/> \
</infosys_refs> \
</entry> \
</entries> \
<condor_tarball arch="default" base_dir="/opt/glidecondor" os="default" \
tar_file="/var/www/html/glidefactory/stage/glidein_v2plus/condor.tgz" version="default"/> \
</glidein>')
valid_file.close()
config_dom = minidom.parse('valid.xml')
# Test valid entries - get all
entries = parse_entries(config_dom, skip_missing_ref_id=False, skip_disabled=False)
os.remove('valid.xml')
self.assertEqual(len(entries), 2)
self.assertEqual(entries['valid_entry']['name'], 'valid_entry')
self.assertEqual(entries['valid_entry']['enabled'], 'True')
self.assertEqual(entries['valid_entry']['gatekeeper'], 'node.fnal.gov/jobmanager-condor')
self.assertEqual(entries['valid_entry']['gridtype'], 'gt2')
self.assertEqual(entries['valid_entry']['work_dir'], 'OSG')
self.assertEqual(entries['valid_entry']['source'], '')
self.assertEqual(entries['valid_entry']['source_type'], '')
self.assertEqual(entries['valid_entry']['glidein_attrs'], {'CONDOR_ARCH':'default', 'GLEXEC_BIN':'NONE', 'GLIDEIN_Site':'VE1'})
self.assertEqual(entries['valid_entry2']['name'], 'valid_entry2')
self.assertEqual(entries['valid_entry2']['enabled'], 'False')
self.assertEqual(entries['valid_entry2']['gatekeeper'], 'node2.fnal.gov/jobmanager-condor')
self.assertEqual(entries['valid_entry2']['gridtype'], 'cream')
self.assertEqual(entries['valid_entry2']['work_dir'], '.')
self.assertEqual(entries['valid_entry2']['source'], 'exp-bdii.cern.ch')
self.assertEqual(entries['valid_entry2']['source_type'], 'BDII')
self.assertEqual(entries['valid_entry2']['glidein_attrs'], {})
# Test with and without rsl
self.assertEqual(entries['valid_entry']['rsl'], '(queue=default)')
self.assertEqual(entries['valid_entry2']['rsl'], '')
# Test with ref id and without ref id
self.assertEqual(entries['valid_entry']['ref_id'], '')
self.assertEqual(entries['valid_entry2']['ref_id'], 'GlueCEUniqueID=node.fnal.gov:2119/jobmanager-condor_default,Mds-Vo-name=TEST,Mds-Vo-name=local,o=grid')
# Test skip missing ref id
missing_ref_id_entries = parse_entries(config_dom, skip_missing_ref_id=True, skip_disabled=False)
self.assertEqual(len(missing_ref_id_entries), 1)
# Test skip disabled
disabled_entries = parse_entries(config_dom, skip_missing_ref_id=False, skip_disabled=True)
self.assertEqual(len(disabled_entries), 1)
# Test skip disabled and missing ref id
disabled_entries = parse_entries(config_dom, skip_missing_ref_id=True, skip_disabled=True)
self.assertEqual(len(disabled_entries), 0)
# Test invalid entry (missing gatekeeper string)
invalid_file = open('invalid.xml', 'w')
invalid_file.write('<glidein schedd_name="*****@*****.**"> \
<entries> \
<entry name="valid_entry" \
enabled="True" \
gridtype="gt2" \
schedd_name="*****@*****.**" \
verbosity="std" \
work_dir="."> \
<infosys_refs> \
</infosys_refs> \
</entry> \
</entries> \
<condor_tarball arch="default" base_dir="/opt/glidecondor" os="default" \
tar_file="/var/www/html/glidefactory/stage/glidein_v2plus/condor.tgz" version="default"/> \
</glidein>')
invalid_file.close()
config_dom = minidom.parse('invalid.xml')
self.assertRaises(KeyError, parse_entries, config_dom, skip_missing_ref_id=False, skip_disabled=False)
os.remove('invalid.xml')
# Test missing entry
missing_entry_file = open('missing_entry.xml', 'w')
missing_entry_file.write('<glidein schedd_name="*****@*****.**"> \
<entries> \
</entries> \
<condor_tarball arch="default" base_dir="/opt/glidecondor" os="default" \
tar_file="/var/www/html/glidefactory/stage/glidein_v2plus/condor.tgz" version="default"/> \
</glidein>')
missing_entry_file.close()
config_dom = minidom.parse('missing_entry.xml')
self.assertRaises(KeyError, parse_entries, config_dom, skip_missing_ref_id=False, skip_disabled=False)
os.remove('missing_entry.xml')
# Test with multiple ref ids (only uses first ref id, for now)
with_ref_ids_file = open('with_ref_ids.xml', 'w')
with_ref_ids_file.write('<glidein schedd_name="*****@*****.**"> \
<entries> \
<entry name="valid_entry" \
enabled="True" \
gatekeeper="node.fnal.gov/jobmanager-condor" \
gridtype="gt2" \
schedd_name="*****@*****.**" \
verbosity="std" \
work_dir="."> \
<infosys_refs> \
<infosys_ref ref="GlueCEUniqueID=node.fnal.gov:2119/jobmanager-condor_default,Mds-Vo-name=TEST,Mds-Vo-name=local,o=grid" server="exp-bdii.cern.ch" type="BDII"/> \
<infosys_ref ref="GlueCEUniqueID=node2.fnal.gov:2119/jobmanager-condor_default,Mds-Vo-name=TEST,Mds-Vo-name=local,o=grid" server="exp-bdii.cern.ch" type="BDII"/> \
</infosys_refs> \
</entry> \
</entries> \
<condor_tarball arch="default" base_dir="/opt/glidecondor" os="default" \
tar_file="/var/www/html/glidefactory/stage/glidein_v2plus/condor.tgz" version="default"/> \
</glidein>')
with_ref_ids_file.close()
config_dom = minidom.parse('with_ref_ids.xml')
entries = parse_entries(config_dom)
self.assertEqual(entries['valid_entry']['ref_id'], 'GlueCEUniqueID=node.fnal.gov:2119/jobmanager-condor_default,Mds-Vo-name=TEST,Mds-Vo-name=local,o=grid')
os.remove('with_ref_ids.xml')