def get_negotiator_status(self):
self.neg_ad = self.coll.locate(htcondor.DaemonTypes.Negotiator, self.pool)
self.neg = htcondor.Negotiator(self.neg_ad)
for entry in self.neg.getPriorities():
for site_info in self.pool_status.values():
grp = entry['Name'].split("@")[0]
if grp in site_info:
site_info[grp]['Priority'] = float(entry['Priority'])
def query_prio_from_neg(coll, negotiator_constraint):
ads = coll.query(htcondor.AdTypes.Negotiator, negotiator_constraint)
if len(ads) == 0:
raise Exception('no negotiator classads found')
elif len(ads) > 1:
logger.warning('multiple negotiators found, using %s' %
(ads[0]['Name']))
n = htcondor.Negotiator(ads[0])
return n.getPriorities()
def get_pool_priorities(pool, retry_delay=30, max_retries=4):
coll = htcondor.Collector(pool)
retries = 0
while retries < max_retries:
try:
ad = coll.locate(htcondor.DaemonTypes.Negotiator)
n = htcondor.Negotiator(ad)
prio = n.getPriorities()
except:
logging.warning("Trouble communicating with pool {0} negotiator, retrying in {1}s.".format(pool,retry_delay))
retries += 1
prio = None
time.sleep(retry_delay)
else:
break
if prio is None:
logging.error("Trouble communicating with pool {0} negotiator, giving up.".format(pool))
return {}
data = {}
for p in prio:
if p['IsAccountingGroup']:
continue
a,schedd = p['Name'].split('@')
parts = a.split('.')
exp = parts[0].split("_")[-1]
name = parts[-1]
basename = "{0}.{1}.{2}".format(
schedd.replace(".","_"),
exp,
name)
for metric in ["ResourcesUsed",
"AccumulatedUsage",
"WeightedAccumulatedUsage",
"Priority",
"WeightedResourcesUsed",
"PriorityFactor"]:
data[basename+"."+metric] = p[metric]
return data
def prepareAcquisition(self):
# Setting defaults
self.source_url = self.config["source_url"]
# Define basic structures
self.condor_projection = [
"JobStatus", "User", "ResidentSetSize", "RequestMemory",
"RequestCpus", "RemoteJob", "GlobalJobId", "RemoteSysCpu",
"RemoteUserCpu", "CurrentTime", "MachineAttrCloudSite0", "QDate",
"JobCurrentStartDate", "JobStartDate", "ServerTime",
"NetworkInputMb", "NetworkOutputMb", "TransferInputSizeMB"
]
self.jobs_status_dict = {
1: "idle",
2: "running",
3: "removed",
4: "completed",
5: "held",
6: "transferred",
7: "suspended"
}
self.jobs_status_colors = [
"#56b4e9", "#009e73", "firebrick", "slateblue", "#d55e00",
"slategrey", "#e69f00"
]
self.quantities_list = [
quantity for quantity in self.condor_projection
if quantity != "GlobalJobId"
]
self.condor_jobs_information = {}
self.user_statistics_dict = {
"removed": 0,
"completed": 0,
"transferred": 0,
"suspended": 0,
"held": 0,
"idle": 0,
"running": 0,
"cores": 0,
"efficiencies": [],
"runtimes": [],
"cputimes": [],
"walltimes": [],
"sites": [],
"ram": 0,
"requested_memory": 0,
"NetworkInputMb": 0.,
"NetworkOutputMb": 0.,
"transfer": 0
}
self.user_statistics = {}
# Prepare htcondor queries
self.collector = htcondor.Collector()
self.schedds = [
htcondor.Schedd(classAd)
for classAd in self.collector.query(htcondor.AdTypes.Schedd)
]
self.queries = []
for schedd in self.schedds:
self.queries.append(
schedd.xquery(
requirements=
"RoutedToJobId =?= undefined && JobUniverse =!= 9",
projection=self.condor_projection))
# Prepare retrieval of user priority information from htcondor
self.negotiator = None
for negotiator_ad in self.collector.query(htcondor.AdTypes.Negotiator):
if negotiator_ad.get("Machine") == "ekpcondorcentral.ekp.kit.edu":
self.negotiator = htcondor.Negotiator(negotiator_ad)
break
if not self.negotiator:
self.negotiator = htcondor.Negotiator()
self.priorities = {}
# Prepare subtable list for database
self.statistics_db_value_list = []