entries_df = datablock.get(data_product_name)
if requests_df.empty or entries_df.empty:
return pandas.DataFrame()
joint_df = requests_df.merge(entries_df,
left_on="ReqName",
right_on="Name",
suffixes=("", "_right"))
requests, entries = split_dataframe(joint_df,
at=len(requests_df.columns))
not_allowed = facts_df.query(
f"fact_name == '{fact_name}' and fact_value == True").empty
if not_allowed:
# Convert request idle to 0
# TODO: Check what to do with max running
# For now keep it same so existing glideins can finish
self.logger.info(
f"Setting ReqIdleGlideins=0 for fact: {allow_type}")
requests.ReqIdleGlideins = 0
return requests
query = self.queries.get(allow_type, "")
if query:
requests.ReqIdleGlideins.where(joint_df.eval(query),
other=0,
inplace=True)
return requests
Publisher.describe(GlideinWMSManifests)
# SPDX-FileCopyrightText: 2017 Fermi Research Alliance, LLC
# SPDX-License-Identifier: Apache-2.0
"""
Publishes price / performance data
"""
import decisionengine_modules.graphite_client as graphite
from decisionengine.framework.modules import Publisher
from decisionengine_modules.graphite.publishers.generic_publisher import GenericPublisher as publisher
@publisher.consumes_dataframe("Nersc_Figure_Of_Merit")
class NerscFigureOfMeritPublisher(publisher):
def __init__(self, config):
super().__init__(config)
def graphite_context(self, dataframe):
self.logger.debug("in NerscFigureOfMeritPublisher graphite_context")
d = {}
for _i, row in dataframe.iterrows():
key = f"{graphite.sanitize_key(row['EntryName'])}.fig_of_merit"
d[key] = row["FigureOfMerit"]
return self.graphite_context_header, d
Publisher.describe(NerscFigureOfMeritPublisher)
# SPDX-FileCopyrightText: 2017 Fermi Research Alliance, LLC
# SPDX-License-Identifier: Apache-2.0
from decisionengine.framework.modules import Publisher
from decisionengine_modules.htcondor.publishers import publisher
@publisher.HTCondorManifests.consumes_dataframes("glideclientglobal_manifests")
class GlideClientGlobalManifests(publisher.HTCondorManifests):
def __init__(self, config):
super().__init__(config)
self.classad_type = "glideclientglobal"
def create_invalidate_constraint(self, dataframe):
self.logger.debug(
"in GlideClientGlobalManifests create_invalidate_constraint")
if not dataframe.empty:
for collector_host, group in dataframe.groupby(["CollectorHost"]):
client_names = list(set(group["ClientName"]))
client_names.sort()
if client_names:
constraint = f"""(glideinmytype == "{self.classad_type}") && (stringlistmember(ClientName, "{','.join(client_names)}"))"""
self.invalidate_ads_constraint[collector_host] = constraint
Publisher.describe(GlideClientGlobalManifests)
"""
Publishes price / performance data
"""
from decisionengine.framework.modules import Publisher
from decisionengine_modules.graphite.publishers.generic_publisher import GenericPublisher as publisher
import decisionengine_modules.graphite_client as graphite
@publisher.consumes_dataframe('GCE_Figure_Of_Merit')
class GCEFigureOfMeritPublisher(publisher):
def __init__(self, config):
super().__init__(config)
def graphite_context(self, dataframe):
d = {}
for i, row in dataframe.iterrows():
key = ('%s.fig_of_merit' %
(graphite.sanitize_key(row['EntryName'])))
d[key] = row['FigureOfMerit']
return self.graphite_context_header, d
Publisher.describe(GCEFigureOfMeritPublisher)
from decisionengine.framework.modules import Publisher
from decisionengine.framework.modules.Publisher import Parameter
@Publisher.supports_config(
Parameter('no_type'),
Parameter('only_type', type=int),
Parameter('default_only', default=2.5),
Parameter('convert_to', type=int,
default=3.2), # Conflicting types: int wins
Parameter('comment', type=str, comment="Single-line comment"),
Parameter('comment_with_nl', type=str, comment="Comment with newline\n"))
class SupportsConfig(Publisher.Publisher):
pass
Publisher.describe(SupportsConfig)
"""
Publishes AWS VM burn rate
"""
from decisionengine.framework.modules import Publisher
from decisionengine_modules.AWS.publishers.AWS_generic_publisher import AWSGenericPublisher as publisher
@publisher.consumes_dataframe('AWS_Burn_Rate')
class AWSBurnRatePublisher(publisher):
def graphite_context(self, dataframe):
d = {}
# There should be only one row [0] in the AWS_Burn_Rate data block
d['FERMILAB.BurnRate'] = dataframe.loc[0, 'BurnRate'].item()
return self.graphite_context_header, d
Publisher.describe(AWSBurnRatePublisher)
# SPDX-FileCopyrightText: 2017 Fermi Research Alliance, LLC
# SPDX-License-Identifier: Apache-2.0
from decisionengine.framework.modules import Publisher
from decisionengine_modules.htcondor.publishers import publisher
@publisher.HTCondorManifests.consumes_dataframes(
"decisionenginemonitor_manifests")
class DecisionEngineMonitorManifests(publisher.HTCondorManifests):
def __init__(self, config):
super().__init__(config)
self.classad_type = "glideclientmonitor"
def create_invalidate_constraint(self, requests_df):
self.logger.debug(
"in DecisionEngineMonitorManifests create_invalidate_constraint")
if not requests_df.empty:
for collector_host, request_group in requests_df.groupby(
["CollectorHost"]):
client_names = list(set(request_group["GlideClientName"]))
client_names.sort()
if client_names:
constraint = f"""(glideinmytype == "{self.classad_type}") && (stringlistmember(GlideClientName, "{','.join(client_names)}"))"""
self.invalidate_ads_constraint[collector_host] = constraint
Publisher.describe(DecisionEngineMonitorManifests)
# SPDX-FileCopyrightText: 2017 Fermi Research Alliance, LLC
# SPDX-License-Identifier: Apache-2.0
import pandas as pd
from decisionengine.framework.modules import Publisher
@Publisher.consumes(bar=pd.DataFrame)
class PublisherNOP(Publisher.Publisher):
def __init__(self, config):
super().__init__(config)
def publish(self, data_block):
self.bar(data_block)
Publisher.describe(PublisherNOP)
"""
Publishes price / performance data
"""
from decisionengine.framework.modules import Publisher
from decisionengine_modules.graphite.publishers.generic_publisher import GenericPublisher as publisher
import decisionengine_modules.graphite_client as graphite
@publisher.consumes_dataframe('GCE_Price_Performance')
class GCEPricePerformancePublisher(publisher):
def __init__(self, config):
super().__init__(config)
def graphite_context(self, dataframe):
d = {}
for i, row in dataframe.iterrows():
key = ('%s.price_perf' % (graphite.sanitize_key(row['EntryName'])))
d[key] = row['PricePerformance']
return self.graphite_context_header, d
Publisher.describe(GCEPricePerformancePublisher)
# SPDX-FileCopyrightText: 2017 Fermi Research Alliance, LLC
# SPDX-License-Identifier: Apache-2.0
"""
Publishes figure of merit data
"""
from decisionengine.framework.modules import Publisher
from decisionengine_modules.AWS.publishers.AWS_generic_publisher import AWSGenericPublisher as publisher
@publisher.consumes_dataframe("AWS_Figure_Of_Merit")
class AWSFOMPublisher(publisher):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
def graphite_context(self, dataframe):
self.logger.debug("in AWSFOMPublisher graphite_context")
d = {}
for _i, row in dataframe.iterrows():
key = f"{row['AccountName']}.{row['EntryName']}.FOM"
d[key] = row["AWS_Figure_Of_Merit"]
return self.graphite_context_header, d
Publisher.describe(AWSFOMPublisher)
# SPDX-FileCopyrightText: 2017 Fermi Research Alliance, LLC
# SPDX-License-Identifier: Apache-2.0
from decisionengine.framework.modules import Publisher
from decisionengine.framework.modules.Publisher import Parameter
@Publisher.supports_config(Parameter("consumes", type=list),
Parameter("expects", type=int))
class DynamicPublisher(Publisher.Publisher):
def __init__(self, config):
self.expects = config["expects"]
self._consumes = dict.fromkeys(config["consumes"], int)
def publisher(self, data_block):
all_values = [data_block[key] for key in self._consumes.keys()]
assert self.expects == sum(all_values, 0)
Publisher.describe(DynamicPublisher)
# SPDX-FileCopyrightText: 2017 Fermi Research Alliance, LLC
# SPDX-License-Identifier: Apache-2.0
"""
Publishes price / performance data
"""
import decisionengine_modules.graphite_client as graphite
from decisionengine.framework.modules import Publisher
from decisionengine_modules.graphite.publishers.generic_publisher import GenericPublisher as publisher
@publisher.consumes_dataframe("job_clusters")
class JobClusteringPublisher(publisher):
def __init__(self, config):
super().__init__(config)
def graphite_context(self, dataframe):
self.logger.debug("in JobClusteringPublisher graphite_context")
d = {}
for _i, row in dataframe.iterrows():
key = f"{graphite.sanitize_key(row['Frontend_Group'])}.job_cluster"
d[key] = row["Totals"]
return self.graphite_context_header, d
Publisher.describe(JobClusteringPublisher)
def wait_for_n_writes(stdout, n):
match = re.search("WriteToDisk:(.*?):", stdout, re.DOTALL)
assert match is not None
with open(match.group(1)) as f:
while True:
lines = f.readlines()
if lines and int(lines[-1].strip()) >= n:
return
time.sleep(1)
@Publisher.supports_config(Parameter("filename", type=str),
Parameter("consumes", type=list))
class WriteToDisk(Publisher.Publisher):
def __init__(self, config):
super().__init__(config)
self.file = tempfile.NamedTemporaryFile("w")
print(f"WriteToDisk:{self.file.name}:")
self.counter = 0
self._consumes = {key: Any for key in config["consumes"]}
self.file.write(f"{self.counter}\n")
self.file.flush()
def publish(self, data_block):
self.counter += 1
self.file.write(f"{self.counter}\n")
self.file.flush()
Publisher.describe(WriteToDisk)
# SPDX-FileCopyrightText: 2017 Fermi Research Alliance, LLC
# SPDX-License-Identifier: Apache-2.0
"""
Publishes GCE VM Burn Rates
"""
from decisionengine.framework.modules import Publisher
from decisionengine_modules.graphite.publishers.generic_publisher import GenericPublisher as publisher
@publisher.consumes_dataframe("GCE_Burn_Rate")
class GCEBurnRatePublisher(publisher):
def __init__(self, config):
super().__init__(config)
def graphite_context(self, dataframe):
self.logger.debug("in GCEBurnRatePublisher graphite_context")
d = {}
# Only one row in this data frame
d["hepcloud-fnal.BurnRate"] = dataframe.loc[0, "BurnRate"].item()
return self.graphite_context_header, d
Publisher.describe(GCEBurnRatePublisher)
@Publisher.consumes(aws_instance_limits=pandas.DataFrame,
spot_occupancy_config=pandas.DataFrame)
class AWSFactoryEntryDataPublisher(Publisher.Publisher):
def __init__(self, config):
super().__init__(config)
self.aws_instance_limits_file = config.get('aws_instance_limits')
self.spot_occupancy_config_file = config.get('spot_occupancy_config')
if None in (self.aws_instance_limits_file, self.spot_occupancy_config_file):
raise RuntimeError('parameters for module config is missing spot_occupancy_config or aws_instance_limits')
def publish(self, datablock):
limits_df = self.aws_instance_limits(datablock)
so_config = self.spot_occupancy_config(datablock).to_dict()
fname = None
with tempfile.NamedTemporaryFile(mode='w', delete=False) as fd:
fname = fd.name
pprint.pprint(so_config, fd)
shutil.move(fname, self.spot_occupancy_config_file)
fname = None
with tempfile.NamedTemporaryFile(mode='w', delete=False) as fd:
fname = fd.name
fd.write('%s' % limits_df.to_csv(index=False))
shutil.move(fname, self.aws_instance_limits_file)
Publisher.describe(AWSFactoryEntryDataPublisher)
# SPDX-FileCopyrightText: 2017 Fermi Research Alliance, LLC
# SPDX-License-Identifier: Apache-2.0
"""
Publishes price / performance data
"""
import decisionengine_modules.graphite_client as graphite
from decisionengine.framework.modules import Publisher
from decisionengine_modules.AWS.publishers.AWS_generic_publisher import AWSGenericPublisher as publisher
@publisher.consumes_dataframe("AWS_Price_Performance")
class AWSPricePerformancePublisher(publisher):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
def graphite_context(self, dataframe):
self.logger.debug("in AWSPricePerformancePublisher graphite_context")
d = {}
for _i, row in dataframe.iterrows():
key = f"{row['AccountName']}.{row['AvailabilityZone']}.{graphite.sanitize_key(row['InstanceType'])}.price_perf"
d[key] = row["AWS_Price_Performance"]
return self.graphite_context_header, d
Publisher.describe(AWSPricePerformancePublisher)