def process_CRIC_cm_dump(self):
"""
Gets the CRIC CM information dump, filters out irrelevant information
and prepares it for insertion into the PanDA DB
"""
data = []
sites_list = dbif.read_configurator_sites(self.session)
for entry in self.CRIC_cm_dump:
_logger.debug('Processing CRIC CM entry {0}'.format(entry))
try:
src = entry['src']
dst = entry['dst']
closeness = entry['closeness']
ts = datetime.now()
# filter out sites that are not in CRIC
skip_sites = []
if src not in sites_list:
skip_sites.append(src)
if dst not in sites_list:
skip_sites.append(dst)
if skip_sites:
_logger.warning("Could not find site(s) {0} in configurator sites".format(skip_sites))
continue
# Skip broken entries (protection against errors in CRIC)
if not src or not dst:
continue
# Prepare data for bulk upserts
data.append((src, dst, 'AGIS_closeness', closeness, ts))
except KeyError:
_logger.warning("CRIC CM entry {0} does not contain one or more of the keys src/dst/closeness".format(entry))
continue
return data
def process_nws_dump(self):
"""
Gets the second generation NWS information dump, filters out irrelevant information
and prepares it for insertion into the PanDA DB
"""
data = []
sites_list = dbif.read_configurator_sites(_session)
# Ignore outdated values
latest_validity = datetime.utcnow() - timedelta(minutes=30)
for src_dst in self.nws_dump:
try:
source, destination = src_dst.split(':')
skip_sites = []
# Skip entries with sites not recognized by configurator
if source not in sites_list:
skip_sites.append(source)
if destination not in sites_list:
skip_sites.append(destination)
if skip_sites:
_logger.warning("Could not find site(s) {0} in configurator sites".format(skip_sites))
continue
except ValueError:
_logger.error("Json wrongly formatted. Expected key with format src:dst, but found key {0}"
.format(src_dst))
continue
# Transferred files
try:
done = self.nws_dump[src_dst][FILES][DONE]
for activity in [PROD_INPUT, PROD_OUTPUT, EXPRESS]:
if activity not in done:
continue
try:
updated_at = datetime.strptime(done[activity][TIMESTAMP], '%Y-%m-%dT%H:%M:%S')
if updated_at > latest_validity:
done_1h = done[activity][H1]
done_6h = done[activity][H6]
data.append((source, destination, activity+'_done_1h', done_1h, updated_at))
data.append((source, destination, activity+'_done_6h', done_6h, updated_at))
except (KeyError, ValueError):
_logger.debug("Entry {0} ({1}->{2}) key {3} does not follow standards"
.format(done, source, destination, activity))
continue
except KeyError:
pass
# Queued files
try:
queued = self.nws_dump[src_dst][FILES][QUEUED]
for activity in [PROD_INPUT, PROD_OUTPUT, EXPRESS]:
if activity not in queued:
continue
try:
updated_at = datetime.strptime(queued[activity][TIMESTAMP], '%Y-%m-%dT%H:%M:%S')
if updated_at > latest_validity:
nqueued = queued[activity][LATEST]
data.append((source, destination, activity+'_queued', nqueued, updated_at))
except (KeyError, ValueError):
_logger.error("Entry {0} ({1}->{2}) key {3} does not follow standards"
.format(queued, source, destination, activity))
continue
except KeyError:
pass
# MBps for Rucio, FAX, PerfSonar
try:
mbps = self.nws_dump[src_dst][MBPS]
for system in mbps:
try:
updated_at = datetime.strptime(mbps[system][TIMESTAMP], '%Y-%m-%dT%H:%M:%S')
except ValueError:
_logger.debug("Entry {0} has wrong timestamp for system {1}".format(mbps, system))
if updated_at > latest_validity:
for duration in [H1, D1, W1]:
try:
mbps_entry = mbps[system][duration]
data.append((source, destination, '{0}_mbps_{1}'.format(system, duration), mbps_entry, updated_at))
except KeyError:
_logger.debug("Entry {0} ({1}->{2}) system {3} duration {4} not available or wrongly formatted"
.format(mbps, source, destination, system, duration))
_logger.debug(sys.exc_info())
except KeyError:
pass
# PerfSonar latency and packetloss
for metric in [LATENCY, PACKETLOSS]:
try:
struc = self.nws_dump[src_dst][metric]
try:
updated_at = datetime.strptime(struc[TIMESTAMP], '%Y-%m-%dT%H:%M:%S')
if updated_at > latest_validity:
value = struc[LATEST]
data.append((source, destination, metric, value, updated_at))
except KeyError:
_logger.debug("Entry {0} ({1}->{2}) does not follow {3} standards"
.format(struc, source, destination, metric))
pass
except KeyError:
continue
return data
def consistency_check(self):
"""
Point out sites, panda sites and DDM endpoints that are missing in one of the sources
"""
# Check for site inconsistencies
agis_sites = set([site['name'] for site in self.site_dump if site['state'] == 'ACTIVE'])
_logger.debug("Sites in AGIS {0}".format(agis_sites))
configurator_sites = dbif.read_configurator_sites(_session)
_logger.debug("Sites in Configurator {0}".format(configurator_sites))
schedconfig_sites = dbif.read_schedconfig_sites(_session)
_logger.debug("Sites in Schedconfig {0}".format(schedconfig_sites))
all_sites = list(agis_sites | configurator_sites | schedconfig_sites)
all_sites.sort()
for site in all_sites:
missing = []
if site not in agis_sites:
missing.append('AGIS')
if site not in configurator_sites:
missing.append('Configurator')
if site not in schedconfig_sites:
missing.append('Schedconfig')
if missing:
_logger.warning("SITE inconsistency: {0} was not found in {1}".format(site, missing))
# Check for panda-site inconsistencies
agis_panda_sites = set([self.schedconfig_dump[long_panda_site_name]['panda_resource']
for long_panda_site_name in self.schedconfig_dump])
_logger.debug("PanDA sites in AGIS {0}".format(agis_panda_sites))
configurator_panda_sites = dbif.read_configurator_panda_sites(_session)
_logger.debug("PanDA sites in Configurator {0}".format(configurator_panda_sites))
schedconfig_panda_sites = dbif.read_schedconfig_panda_sites(_session)
_logger.debug("PanDA sites in Schedconfig {0}".format(schedconfig_panda_sites))
all_panda_sites = list(agis_panda_sites | configurator_panda_sites | schedconfig_panda_sites)
all_panda_sites.sort()
for site in all_panda_sites:
missing = []
if site not in agis_panda_sites:
missing.append('AGIS')
if site not in configurator_panda_sites:
missing.append('Configurator')
if site not in schedconfig_panda_sites:
missing.append('Schedconfig')
if missing:
_logger.warning("PanDA SITE inconsistency: {0} was not found in {1}".format(site, missing))
# Check for DDM endpoint inconsistencies
agis_ddm_endpoints = set([ddm_endpoint_name for ddm_endpoint_name in self.endpoint_token_dict])
_logger.debug("DDM endpoints in AGIS {0}".format(agis_ddm_endpoints))
configurator_ddm_endpoints = dbif.read_configurator_ddm_endpoints(_session)
_logger.debug("DDM endpoints in Configurator {0}".format(configurator_ddm_endpoints))
all_ddm_endpoints = list(agis_ddm_endpoints | configurator_ddm_endpoints)
all_ddm_endpoints.sort()
for site in all_ddm_endpoints:
missing = []
if site not in agis_ddm_endpoints:
missing.append('AGIS')
if site not in configurator_ddm_endpoints:
missing.append('Configurator')
if missing:
_logger.warning("DDM ENDPOINT inconsistency: {0} was not found in {1}".format(site, missing))
self.cleanup_configurator(agis_sites, agis_panda_sites, agis_ddm_endpoints, configurator_sites,
configurator_panda_sites, configurator_ddm_endpoints)