def transform(self, datablock): """ Grid sites FOM are straight up assumed as 0 for now """ self.logger.debug("in GridFigureOfMerit transform") entries = self.Factory_Entries_Grid(datablock) if entries is None: entries = pandas.DataFrame({ATTR_ENTRYNAME: []}) foms = [] if not entries.empty: for _index, entry in entries.iterrows(): running = float(entry["GlideinMonitorTotalStatusRunning"]) max_allowed = float(entry["GlideinConfigPerEntryMaxGlideins"]) max_idle = float(entry["GlideinConfigPerEntryMaxIdle"]) idle = float(entry["GlideinMonitorTotalStatusIdle"]) f = { ATTR_ENTRYNAME: entry[ATTR_ENTRYNAME], ATTR_FOM: figure_of_merit(self.price_performance, running, max_allowed, idle, max_idle, self.logger), } foms.append(f) return {"Grid_Figure_Of_Merit": pandas.DataFrame(foms)}
def transform(self, data_block): """ NERSC Instance performance is obtained from the following CSV: InstanceType,AvailabilityZone,OnDemandPrice,PerfTtbarTotal,EntryName haswell,cori,0.576,0.96,CMSHTPC_T3_US_NERSC_Cori haswell_shared,cori,0.081,0.137,CMSHTPC_T3_US_NERSC_Cori_shared knl,cori,0.612,0.255,CMSHTPC_T3_US_NERSC_Cori_KNL haswell,edison,0.432,0.658,CMSHTPC_T3_US_NERSC_Edison haswell_shared,edison,0.108,0.164,CMSHTPC_T3_US_NERSC,Edison_shared by NerscInstancePerformance first column is HW type, second the machine, third the \"price\" of what 1 hr of that resource is worth 4th column is the cms ttbar benchmark price/performance is the ratio of the 3rd and 4th column """ self.logger.debug("in NerscFigureOfMerit transform") performance = self.Nersc_Instance_Performance(data_block) performance["PricePerformance"] = np.where( performance["PerfTtbarTotal"] > 0, (performance["OnDemandPrice"] / performance["PerfTtbarTotal"]), sys.float_info.max, ) factory_entries_lcf = self.Factory_Entries_LCF(data_block) figures_of_merit = [] for _i, row in factory_entries_lcf.iterrows(): entry_name = row["EntryName"] perf_df = performance[performance.EntryName == entry_name] for _j, perf_row in perf_df.iterrows(): running = float(row["GlideinMonitorTotalStatusRunning"]) max_allowed = float(row["GlideinConfigPerEntryMaxGlideins"]) max_idle = float(row["GlideinConfigPerEntryMaxIdle"]) idle = float(row["GlideinMonitorTotalStatusIdle"]) figures_of_merit.append({ "EntryName": entry_name, "FigureOfMerit": fom.figure_of_merit(perf_row["PricePerformance"], running, max_allowed, idle, max_idle, self.logger), }) return { "Nersc_Price_Performance": performance.filter(["EntryName", "PricePerformance"]), "Nersc_Figure_Of_Merit": pd.DataFrame(figures_of_merit), }
def transform(self, data_block): self.logger.debug("in GceFigureOfMerit transform") performance = self.GCE_Instance_Performance(data_block) performance["PricePerformance"] = np.where( performance["PerfTtbarTotal"] > 0, (performance["OnDemandPrice"] / performance["PerfTtbarTotal"]), sys.float_info.max, ) factory_entries = self.Factory_Entries_GCE(data_block).fillna(0) gce_occupancy = self.GCE_Occupancy(data_block).fillna(0) figures_of_merit = [] for _i, row in performance.iterrows(): az = row["AvailabilityZone"] it = row["InstanceType"] entry_name = row["EntryName"] occupancy_df = gce_occupancy[( (gce_occupancy.AvailabilityZone == az) & (gce_occupancy.InstanceType == it))] occupancy = float(occupancy_df["Occupancy"].values[0] ) if not occupancy_df.empty else 0 max_allowed = max_idle = idle = 0 if (not factory_entries.empty) and ("EntryName" in factory_entries): factory_df = factory_entries[factory_entries.EntryName == entry_name] if not factory_df.empty: max_allowed = float( factory_df["GlideinConfigPerEntryMaxGlideins"]. values[0]) max_idle = float( factory_df["GlideinConfigPerEntryMaxIdle"].values[0]) idle = float( factory_df["GlideinMonitorTotalStatusIdle"].values[0]) fom = figure_of_merit(row["PricePerformance"], occupancy, max_allowed, idle, max_idle, self.logger) figures_of_merit.append({ "EntryName": entry_name, "FigureOfMerit": fom }) return { "GCE_Price_Performance": performance.filter(["EntryName", "PricePerformance"]), "GCE_Figure_Of_Merit": pd.DataFrame(figures_of_merit), }
def transform(self, data_block): performance = data_block[CONSUMES[0]] performance["PricePerformance"] = np.where( performance["PerfTtbarTotal"] > 0, (performance["OnDemandPrice"] / performance["PerfTtbarTotal"]), sys.float_info.max) factory_entries = data_block[CONSUMES[1]].fillna(0) gce_occupancy = data_block[CONSUMES[2]].fillna(0) figures_of_merit = [] for i, row in performance.iterrows(): az = row["AvailabilityZone"] it = row["InstanceType"] entry_name = row["EntryName"] occupancy_df = gce_occupancy[( (gce_occupancy.AvailabilityZone == az) & (gce_occupancy.InstanceType == it))] occupancy = float(occupancy_df["Occupancy"].values[0] ) if not occupancy_df.empty else 0 max_allowed = max_idle = idle = 0 if (not factory_entries.empty) and ('EntryName' in factory_entries): factory_df = factory_entries[factory_entries.EntryName == entry_name] if not factory_df.empty: max_allowed = float( factory_df["GlideinConfigPerEntryMaxGlideins"]. values[0]) max_idle = float( factory_df["GlideinConfigPerEntryMaxIdle"].values[0]) idle = float( factory_df["GlideinMonitorTotalStatusIdle"].values[0]) fom = figure_of_merit(row["PricePerformance"], occupancy, max_allowed, idle, max_idle) figures_of_merit.append({ "EntryName": entry_name, "FigureOfMerit": fom }) return { PRODUCES[0]: performance.filter(["EntryName", "PricePerformance"]), PRODUCES[1]: pd.DataFrame(figures_of_merit) }
def transform(self, datablock): """ Grid sites FOM are straight up assumed as 0 for now """ entries = datablock.get('Factory_Entries_Grid', pandas.DataFrame({ATTR_ENTRYNAME: []})) foms = [] if not entries.empty: for index, entry in entries.iterrows(): running = float(entry['GlideinMonitorTotalStatusRunning']) max_allowed = float(entry['GlideinConfigPerEntryMaxGlideins']) max_idle = float(entry['GlideinConfigPerEntryMaxIdle']) idle = float(entry['GlideinMonitorTotalStatusIdle']) f = { ATTR_ENTRYNAME: entry[ATTR_ENTRYNAME], ATTR_FOM: figure_of_merit(self.price_performance, running, max_allowed, idle, max_idle) } foms.append(f) return {PRODUCES[0]: pandas.DataFrame(foms)}