def test_eligible_resources_with_constraints_limit():
test_df = pandas.DataFrame({
"EntryName": ["g1", "g2", "g3", "g4", "g5", "a1", "n5", "a2"],
"FOM": [0.00, 0.00, 0.00, 0.00, 0.00, 0.01, 0.05, 0.10],
})
fom_df = resource_dist_plugins.fom_eligible_resources(
resources, constraint="FOM < 1", limit=8).reset_index(drop=True)
assert test_df.equals(fom_df)
def test_eligible_resources_with_constraints_limit(self):
test_df = pandas.DataFrame({
'EntryName': ['g1', 'g2', 'g3', 'g4', 'g5', 'a1', 'n5', 'a2'],
'FOM': [0.00, 0.00, 0.00, 0.00, 0.00, 0.01, 0.05, 0.10]
})
fom_df = resource_dist_plugins.fom_eligible_resources(
resources, constraint='FOM < 1', limit=8).reset_index(drop=True)
assert test_df.equals(fom_df)
def test_eligible_resources():
test_df = pandas.DataFrame({
"EntryName": [
"g1", "g2", "g3", "g4", "g5", "a1", "n5", "a2", "a4", "n1", "a5",
"n2", "a3", "n3", "n4"
],
"FOM": [
0.00, 0.00, 0.00, 0.00, 0.00, 0.01, 0.05, 0.10, 0.10, 0.10, 0.15,
0.30, 1.00, 1.00, 2.10
],
})
eligible_resources = resource_dist_plugins.fom_eligible_resources(
resources)
assert test_df.equals(eligible_resources)
def test_eligible_resources(self):
test_df = pandas.DataFrame({
'EntryName': [
'g1', 'g2', 'g3', 'g4', 'g5', 'a1', 'n5', 'a2', 'a4', 'n1',
'a5', 'n2', 'a3', 'n3', 'n4'
],
'FOM': [
0.00, 0.00, 0.00, 0.00, 0.00, 0.01, 0.05, 0.10, 0.10, 0.10,
0.15, 0.30, 1.00, 1.00, 2.10
]
})
eligible_resources = resource_dist_plugins.fom_eligible_resources(
resources)
assert test_df.equals(eligible_resources)
def transform(self, datablock):
"""
Make all necessary calculations
:type datablock: :obj:`~datablock.DataBlock`
:arg datablock: data block
:rtype: pandas frame (:class:`pd.DataFramelist`)
"""
# Dict to be returned
manifests = {}
try:
# Get the frontend config dict
fe_cfg = self.read_fe_config()
# Get factory global classad dataframe
factory_globals = datablock.get('factoryglobal_manifests')
entries = pandas.DataFrame(
pandas.concat(
[datablock.get(et) for et in _SUPPORTED_ENTRY_TYPES],
ignore_index=True,
sort=True))
if entries.empty:
self.logger.info(
'There are no entries to request resources from')
return dict.fromkeys(
['glideclientglobal_manifests', 'glideclient_manifests'],
pandas.DataFrame())
# Sanitize 'auto' in GLIDEIN_CPUS and convert it to a valid int
entries = self.sanitize_entries(entries)
# Shortlisted entries using Figure of Merit
# TODO: This will be influenced once we can configure different
# resource selection plugins. Currently supports FOM only.
foms = {
'Grid_Figure_Of_Merit': self.Grid_Figure_Of_Merit(),
'GCE_Figure_Of_Merit': self.GCE_Figure_Of_Merit(),
'AWS_Figure_Of_Merit': self.AWS_Figure_Of_Merit(),
'Nersc_Figure_Of_Merit': self.Nersc_Figure_Of_Merit()
}
fom_entries = fom_eligible_resources(
foms,
constraint=self.fom_resource_constraint,
limit=self.fom_resource_limit)
self.logger.debug('Figure of Merits')
self.logger.debug(fom_entries)
# Get the jobs dataframe
jobs_df = self.job_manifests()
# Get the job clusters dataframe
job_clusters_df = self.job_clusters()
# Get HTCondor slots dataframe
slots_df = self.startd_manifests()
# self.logger.info(job_clusters_df)
for index, row in job_clusters_df.iterrows():
# Each job bucket represents a frontend group equivalent
# For every job bucket figure out how many glideins to request
# at which entry (i.e entries matching entry query expressions)
self.logger.info(
'--------------------------------------------')
fe_group = row.get('Frontend_Group')
self.logger.info(
'Processing glidein requests for the FE Group: %s' %
fe_group)
job_query = row.get('Job_Bucket_Criteria_Expr')
self.logger.info('Frontend Group %s job query: %s' %
(fe_group, job_query))
match_exp = ' or '.join(row.get('Site_Bucket_Criteria_Expr'))
self.logger.info(
'Frontend Group %s site matching expression : %s' %
(fe_group, match_exp))
self.logger.info(
'--------------------------------------------')
matched_entries = entries.query(match_exp)
# Get the Frontend element object. Currently FOM.
gfe = glide_frontend_element.get_gfe_obj(
fe_group, self.acct_group, fe_cfg)
# Generate glideclient and glideclientglobal manifests
# for this bucket/frontend group
group_manifests = \
gfe.generate_glidein_requests(
# jobs_df, job_clusters_df, slots_df, matched_entries,
jobs_df, slots_df, matched_entries, factory_globals,
job_filter=job_query, fom_entries=fom_entries)
manifests = self.merge_requests(manifests, group_manifests)
except Exception:
self.logger.error('Error generating glidein requests: %s' %
traceback.format_exc())
raise
return manifests
