def transform(self, datablock):
"""
Grid sites FOM are straight up assumed as 0 for now
"""
self.logger.debug("in GridFigureOfMerit transform")
entries = self.Factory_Entries_Grid(datablock)
if entries is None:
entries = pandas.DataFrame({ATTR_ENTRYNAME: []})
foms = []
if not entries.empty:
for _index, entry in entries.iterrows():
running = float(entry["GlideinMonitorTotalStatusRunning"])
max_allowed = float(entry["GlideinConfigPerEntryMaxGlideins"])
max_idle = float(entry["GlideinConfigPerEntryMaxIdle"])
idle = float(entry["GlideinMonitorTotalStatusIdle"])
f = {
ATTR_ENTRYNAME:
entry[ATTR_ENTRYNAME],
ATTR_FOM:
figure_of_merit(self.price_performance, running,
max_allowed, idle, max_idle, self.logger),
}
foms.append(f)
return {"Grid_Figure_Of_Merit": pandas.DataFrame(foms)}
def transform(self, data_block):
"""
NERSC Instance performance is obtained from the following CSV:
InstanceType,AvailabilityZone,OnDemandPrice,PerfTtbarTotal,EntryName
haswell,cori,0.576,0.96,CMSHTPC_T3_US_NERSC_Cori
haswell_shared,cori,0.081,0.137,CMSHTPC_T3_US_NERSC_Cori_shared
knl,cori,0.612,0.255,CMSHTPC_T3_US_NERSC_Cori_KNL
haswell,edison,0.432,0.658,CMSHTPC_T3_US_NERSC_Edison
haswell_shared,edison,0.108,0.164,CMSHTPC_T3_US_NERSC,Edison_shared
by NerscInstancePerformance
first column is HW type, second the machine, third the \"price\"
of what 1 hr of that resource is worth
4th column is the cms ttbar benchmark
price/performance is the ratio of the 3rd and 4th column
"""
self.logger.debug("in NerscFigureOfMerit transform")
performance = self.Nersc_Instance_Performance(data_block)
performance["PricePerformance"] = np.where(
performance["PerfTtbarTotal"] > 0,
(performance["OnDemandPrice"] / performance["PerfTtbarTotal"]),
sys.float_info.max,
)
factory_entries_lcf = self.Factory_Entries_LCF(data_block)
figures_of_merit = []
for _i, row in factory_entries_lcf.iterrows():
entry_name = row["EntryName"]
perf_df = performance[performance.EntryName == entry_name]
for _j, perf_row in perf_df.iterrows():
running = float(row["GlideinMonitorTotalStatusRunning"])
max_allowed = float(row["GlideinConfigPerEntryMaxGlideins"])
max_idle = float(row["GlideinConfigPerEntryMaxIdle"])
idle = float(row["GlideinMonitorTotalStatusIdle"])
figures_of_merit.append({
"EntryName":
entry_name,
"FigureOfMerit":
fom.figure_of_merit(perf_row["PricePerformance"], running,
max_allowed, idle, max_idle,
self.logger),
})
return {
"Nersc_Price_Performance":
performance.filter(["EntryName", "PricePerformance"]),
"Nersc_Figure_Of_Merit":
pd.DataFrame(figures_of_merit),
}
def transform(self, data_block):
self.logger.debug("in GceFigureOfMerit transform")
performance = self.GCE_Instance_Performance(data_block)
performance["PricePerformance"] = np.where(
performance["PerfTtbarTotal"] > 0,
(performance["OnDemandPrice"] / performance["PerfTtbarTotal"]),
sys.float_info.max,
)
factory_entries = self.Factory_Entries_GCE(data_block).fillna(0)
gce_occupancy = self.GCE_Occupancy(data_block).fillna(0)
figures_of_merit = []
for _i, row in performance.iterrows():
az = row["AvailabilityZone"]
it = row["InstanceType"]
entry_name = row["EntryName"]
occupancy_df = gce_occupancy[(
(gce_occupancy.AvailabilityZone == az) &
(gce_occupancy.InstanceType == it))]
occupancy = float(occupancy_df["Occupancy"].values[0]
) if not occupancy_df.empty else 0
max_allowed = max_idle = idle = 0
if (not factory_entries.empty) and ("EntryName"
in factory_entries):
factory_df = factory_entries[factory_entries.EntryName ==
entry_name]
if not factory_df.empty:
max_allowed = float(
factory_df["GlideinConfigPerEntryMaxGlideins"].
values[0])
max_idle = float(
factory_df["GlideinConfigPerEntryMaxIdle"].values[0])
idle = float(
factory_df["GlideinMonitorTotalStatusIdle"].values[0])
fom = figure_of_merit(row["PricePerformance"], occupancy,
max_allowed, idle, max_idle, self.logger)
figures_of_merit.append({
"EntryName": entry_name,
"FigureOfMerit": fom
})
return {
"GCE_Price_Performance":
performance.filter(["EntryName", "PricePerformance"]),
"GCE_Figure_Of_Merit":
pd.DataFrame(figures_of_merit),
}
def transform(self, data_block):
performance = data_block[CONSUMES[0]]
performance["PricePerformance"] = np.where(
performance["PerfTtbarTotal"] > 0,
(performance["OnDemandPrice"] / performance["PerfTtbarTotal"]),
sys.float_info.max)
factory_entries = data_block[CONSUMES[1]].fillna(0)
gce_occupancy = data_block[CONSUMES[2]].fillna(0)
figures_of_merit = []
for i, row in performance.iterrows():
az = row["AvailabilityZone"]
it = row["InstanceType"]
entry_name = row["EntryName"]
occupancy_df = gce_occupancy[(
(gce_occupancy.AvailabilityZone == az) &
(gce_occupancy.InstanceType == it))]
occupancy = float(occupancy_df["Occupancy"].values[0]
) if not occupancy_df.empty else 0
max_allowed = max_idle = idle = 0
if (not factory_entries.empty) and ('EntryName'
in factory_entries):
factory_df = factory_entries[factory_entries.EntryName ==
entry_name]
if not factory_df.empty:
max_allowed = float(
factory_df["GlideinConfigPerEntryMaxGlideins"].
values[0])
max_idle = float(
factory_df["GlideinConfigPerEntryMaxIdle"].values[0])
idle = float(
factory_df["GlideinMonitorTotalStatusIdle"].values[0])
fom = figure_of_merit(row["PricePerformance"], occupancy,
max_allowed, idle, max_idle)
figures_of_merit.append({
"EntryName": entry_name,
"FigureOfMerit": fom
})
return {
PRODUCES[0]: performance.filter(["EntryName", "PricePerformance"]),
PRODUCES[1]: pd.DataFrame(figures_of_merit)
}
def transform(self, datablock):
"""
Grid sites FOM are straight up assumed as 0 for now
"""
entries = datablock.get('Factory_Entries_Grid',
pandas.DataFrame({ATTR_ENTRYNAME: []}))
foms = []
if not entries.empty:
for index, entry in entries.iterrows():
running = float(entry['GlideinMonitorTotalStatusRunning'])
max_allowed = float(entry['GlideinConfigPerEntryMaxGlideins'])
max_idle = float(entry['GlideinConfigPerEntryMaxIdle'])
idle = float(entry['GlideinMonitorTotalStatusIdle'])
f = {
ATTR_ENTRYNAME: entry[ATTR_ENTRYNAME],
ATTR_FOM: figure_of_merit(self.price_performance,
running, max_allowed,
idle, max_idle)
}
foms.append(f)
return {PRODUCES[0]: pandas.DataFrame(foms)}
