def find_new_entries_in_infosys(config_xml, source, source_type, skip_disabled, vo_name=''):
"""
For the given information system, find any new entries that are not already in the config.
"""
try:
# Find all config entries not disabled
config_dom = minidom.parse(config_xml)
config_entries = infosys_lib.parse_entries(config_dom, skip_missing_ref_id=True, skip_disabled=skip_disabled)
except:
print "Error parsing the config file '%s' for entries, exiting the tool." % config_xml
sys.exit(2)
# Query the given info system
if source_type.lower() == 'bdii':
infosys_entries = infosys_lib.query_bdii(source, vo_name)
elif source_type.lower() == 'ress':
# Update path with condor
condor_path = infosys_lib.parse_condor_path(config_dom)
os.environ["CONDOR_CONFIG"] = condor_path + "/etc/condor_config"
condorExe.set_path(condor_path + "/bin", condor_path + "/sbin")
ress_entries = infosys_lib.query_ress(source, vo_name)
# Remove duplicate entries
infosys_entries = remove_duplicates(ress_entries)
elif source_type.lower() == 'tg':
infosys_entries = infosys_lib.query_teragrid()
# Compare config entries with what is found in the information system
new_entries = []
for infosys_id in infosys_entries:
entry_i = infosys_entries[infosys_id]
found_match = False
for config_entry in config_entries:
entry_c = config_entries[config_entry]
# Check if ids match
if entry_i['ref_id'] == entry_c['ref_id']:
# Check same source types between config and infosys entries
# TODO do we need to check source url too?
if entry_c['source_type'].lower() == source_type.lower():
found_match = True # already have this entry
break
else:
# Check if content matches for other infosys or manual entries
if entry_i['gatekeeper'] == entry_c['gatekeeper'] and entry_i['gridtype'] == entry_c['gridtype'] and entry_i['rsl'] == entry_c['rsl']:
found_match = True # already have this entry
# TODO here could add ability to update ref_ids if find additional matching entry
# not sure if we want to for ress entries tho?
break
if not found_match:
new_entries.append(infosys_entries[infosys_id])
return new_entries
def test_parse_condor_path(self):
"""
Test that a valid condor path can be found in the factory config dom.
"""
# Test valid entries
valid_file = open('valid.xml', 'w')
valid_file.write('<glidein schedd_name="[email protected],[email protected]"> \
<entries> \
<entry name="valid_entry" \
enabled="True" \
gatekeeper="node.fnal.gov/jobmanager-condor" \
gridtype="gt2" \
rsl="(queue=default)" \
schedd_name="*****@*****.**" \
verbosity="std" \
work_dir="OSG"> \
<attrs> \
</attrs> \
<infosys_refs> \
</infosys_refs> \
</entry> \
</entries> \
<condor_tarball arch="default" base_dir="/opt/glidecondor" os="default" \
tar_file="/var/www/html/glidefactory/stage/glidein_v2plus/condor.tgz" version="default"/> \
</glidein>')
valid_file.close()
config_dom = minidom.parse('valid.xml')
condor_path = parse_condor_path(config_dom)
self.assertEqual(condor_path, "/opt/glidecondor")
os.remove('valid.xml')
# Test missing condor path
invalid_condor_file = open('invalid_condor.xml', 'w')
invalid_condor_file.write('<glidein schedd_name="*****@*****.**"> \
<entries> \
<entry name="valid_entry" \
enabled="True" \
gridtype="gt2" \
schedd_name="*****@*****.**" \
verbosity="std" \
work_dir="."> \
<infosys_refs> \
</infosys_refs> \
</entry> \
</entries> \
<condor_tarball arch="default" os="default" \
tar_file="/var/www/html/glidefactory/stage/glidein_v2plus/condor.tgz" version="default"/> \
</glidein>')
invalid_condor_file.close()
config_dom = minidom.parse('invalid_condor.xml')
self.assertRaises(KeyError, parse_condor_path, config_dom)
os.remove('invalid_condor.xml')
def find_entries_with_partial_id_match(config_xml, skip_disabled):
"""
Finds the bdii, ress and TeraGrid entries with partial matches.
"""
try:
# Find all enabled config entries with ref ids
config_dom = minidom.parse(config_xml)
config_entries = infosys_lib.parse_entries(config_dom, skip_missing_ref_id=True, skip_disabled=skip_disabled)
# Create an info systems list from factory config
infosystems = infosys_lib.parse_info_systems(config_dom)
has_ress = False
for infosys in infosystems:
if infosystems[infosys].lower() == 'ress':
has_ress = True
break
if has_ress:
# Update path with condor
condor_path = infosys_lib.parse_condor_path(config_dom)
os.environ["CONDOR_CONFIG"] = condor_path + "/etc/condor_config"
condorExe.set_path(condor_path + "/bin", condor_path + "/sbin")
except:
print "Error parsing the the config file '%s', exiting the tool." % config_xml
sys.exit(2)
# Retrieve info systems entries
bdii_entries = {}
ress_entries = {}
tg_entries = {}
for infosys, type in infosystems.iteritems():
if type.lower() == 'bdii':
bdii_entries.update(infosys_lib.query_bdii(infosys))
elif type.lower() == 'ress':
ress_entries.update(infosys_lib.query_ress(infosys))
elif type.lower() == 'tg':
tg_entries.update(infosys_lib.query_teragrid(infosys))
partial_match_bdii_entries = find_partial_id_match(bdii_entries, config_entries, 'bdii')
partial_match_ress_entries = find_partial_id_match(ress_entries, config_entries, 'ress')
partial_match_tg_entries = find_partial_id_match(tg_entries, config_entries, 'tg')
return partial_match_bdii_entries, partial_match_ress_entries, partial_match_tg_entries
