def test_acquire(self):
nersc_job_info = NerscJobInfo.NerscJobInfo(CONFIG)
with mock.patch.object(newt.Newt, "get_status") as get_status:
get_status.return_value = utils.input_from_file(
STATUS_FIXTURE_FILE)
with mock.patch.object(newt.Newt, "get_queue") as get_queue:
get_queue.return_value = utils.input_from_file(
JOBS_FIXTURE_FILE)
res = nersc_job_info.acquire()
assert PRODUCES == list(res.keys())
assert EXPECTED_PANDAS_DFRAME.equals(res[PRODUCES[0]])
def test_acquire():
nersc_job_info = NerscJobInfo.NerscJobInfo(CONFIG)
with mock.patch.object(newt.Newt, "get_status") as get_status:
get_status.return_value = utils.input_from_file(STATUS_FIXTURE_FILE)
with mock.patch.object(newt.Newt, "get_queue") as get_queue:
get_queue.return_value = utils.input_from_file(JOBS_FIXTURE_FILE)
res = nersc_job_info.acquire()
verify_products(nersc_job_info, res)
new_df = res["Nersc_Job_Info"]
new_df = new_df.reindex(EXPECTED_PANDAS_DFRAME.columns, axis=1)
pandas.testing.assert_frame_equal(EXPECTED_PANDAS_DFRAME, new_df)
def test_acquire(self):
nersc_job_info = NerscJobInfo.NerscJobInfo(CONFIG)
with mock.patch.object(newt.Newt, "get_status") as get_status:
get_status.return_value = utils.input_from_file(
STATUS_FIXTURE_FILE)
with mock.patch.object(newt.Newt, "get_queue") as get_queue:
get_queue.return_value = utils.input_from_file(
JOBS_FIXTURE_FILE)
res = nersc_job_info.acquire()
assert PRODUCES == list(res.keys())
new_df = res[PRODUCES[0]]
new_df = new_df.reindex(EXPECTED_PANDAS_DFRAME.columns, axis=1)
pandas.testing.assert_frame_equal(EXPECTED_PANDAS_DFRAME,
new_df)
def test_acquire(self):
nersc_allocations = NerscAllocationInfo.NerscAllocationInfo(CONFIG)
with mock.patch.object(newt.Newt, "get_usage") as f:
f.return_value = utils.input_from_file(ALLOCATIONS_FIXTURE_FILE)
res = nersc_allocations.acquire()
assert PRODUCES == list(res.keys())
assert EXPECTED_PANDAS_DFRAME.equals(res[PRODUCES[0]])
def test_condorstatus(self):
condor_status = htcondor_query.CondorStatus(
subsystem_name="any", pool_name=config_cs.get("pool_name"))
with mock.patch.object(htcondor_query.CondorStatus, "fetch") as f:
f.return_value = utils.input_from_file(FIXTURE_FILE)
condor_status.load()
assert f.return_value == condor_status.stored_data
def test_condorq(self):
condor_q = htcondor_query.CondorQ(
schedd_name=config_cq.get("schedd_name"),
pool_name=config_cq.get("pool_name"))
with mock.patch.object(htcondor_query.CondorQ, "fetch") as f:
f.return_value = utils.input_from_file(FIXTURE_FILE)
condor_q.load()
assert f.return_value == condor_q.stored_data
def test_acquire():
with mock.patch.object(configmanager, "ChannelConfigHandler"), \
mock.patch.object(dataspace, "DataSpace"), \
mock.patch.object(SourceProxy.SourceProxy, "acquire") as factory_data:
gce_resource_limits = GCEResourceLimits.GCEResourceLimits(CONFIG)
factory_entries = utils.input_from_file(FIXTURE_FILE)
factory_data.return_value = {"Factory_Entries_GCE": pandas.DataFrame(factory_entries)}
gce_limits = gce_resource_limits.acquire()
assert _PRODUCES.keys() == gce_limits.keys()
assert CONFIG.get("entry_limit_attrs").sort() == list(gce_limits.get('GCE_Resource_Limits')).sort()
def test_acquire_correctionmap():
df1 = pd.DataFrame(data={"GLIDEIN_Resource_Slots": ["DummySlots", "DummySlots"]})
df2 = pd.DataFrame(data={"GLIDEIN_CMSSite": ["DummySite", "DummySite"]})
entries = factory_entries.FactoryEntries(CONFIG_FACTORY_ENTRIES_CORMAP)
with mock.patch.object(htcondor_query.CondorStatus, "fetch") as f:
f.return_value = utils.input_from_file(FIXTURE_FILE)
dummypd = entries.acquire()
dummypd2 = dummypd["Factory_Entries_Grid"]
assert df1.equals(dummypd2[["GLIDEIN_Resource_Slots"]])
assert df2.equals(dummypd2[["GLIDEIN_CMSSite"]])
def test_transform(self):
with mock.patch.object(Transform.Transform, "__init__", transform_init_mock):
aws_s_p = AWSSpotPrice.AWSSpotPrice(config)
with mock.patch.object(boto3.session, "Session") as s:
s.return_value = SessionMock()
with mock.patch.object(AWSSpotPrice.AWSSpotPriceForRegion, "get_price") as get_price:
sp_d = utils.input_from_file(os.path.join(DATA_DIR, "spot_price.fixture"))
get_price.return_value = sp_d
res = aws_s_p.transform(account)
assert produces.keys() == res.keys()
new_df = fix_spot_price(res["provisioner_resource_spot_prices"])
expected_pandas_df2 = expected_pandas_df.astype("object")
pd.testing.assert_frame_equal(expected_pandas_df2, new_df)
def test_acquire(self):
with mock.patch.object(SourceProxy.SourceProxy, "__init__",
lambda x, y: None):
aws_s_p = AWSSpotPrice.AWSSpotPrice(config)
with mock.patch.object(SourceProxy.SourceProxy,
'acquire') as acquire:
acquire.return_value = account
with mock.patch.object(boto3.session, 'Session') as s:
s.return_value = SessionMock()
with mock.patch.object(AWSSpotPrice.AWSSpotPriceForRegion,
'get_price') as get_price:
sp_d = utils.input_from_file(
os.path.join(DATA_DIR, 'spot_price.fixture'))
get_price.return_value = sp_d
res = aws_s_p.acquire()
assert produces == list(res.keys())
new_df = fix_spot_price(res[produces[0]])
assert utils.compare_dfs(expected_pandas_df, new_df)
def test_acquire(self):
with mock.patch.object(SourceProxy.SourceProxy, "__init__", lambda x, y: None):
aws_occ = Occupancy.AWSOccupancy(config)
with mock.patch.object(SourceProxy.SourceProxy, 'acquire') as acquire:
acquire.return_value = account
with mock.patch.object(boto3.session, 'Session') as s:
s.return_value = SessionMock()
with mock.patch.object(Occupancy.OccupancyForRegion, 'get_ec2_instances') as get_instances:
cap = utils.input_from_file(os.path.join(DATA_DIR,
'occupancy.fixture'))
get_instances.return_value = cap
res = aws_occ.acquire()
assert produces == list(res.keys())
df1 = expected_pandas_df.sort_values(
['AvailabilityZone', 'InstanceType'])
new_df = res.get(produces[0]).sort_values(
['AvailabilityZone', 'InstanceType'])
new_df.reindex()
assert utils.compare_dfs(df1, new_df)
def test_transform(self):
with mock.patch.object(Transform.Transform, "__init__",
transform_init_mock):
aws_occ = Occupancy.AWSOccupancy(config)
with mock.patch.object(boto3.session, "Session") as s:
s.return_value = SessionMock()
with mock.patch.object(Occupancy.OccupancyForRegion,
"get_ec2_instances") as get_instances:
cap = utils.input_from_file(
os.path.join(DATA_DIR, "occupancy.fixture"))
get_instances.return_value = cap
res = aws_occ.transform(account)
assert produces.keys() == res.keys()
df1 = expected_pandas_df.sort_values(
["AvailabilityZone", "InstanceType"])
new_df = res.get("AWS_Occupancy").sort_values(
["AvailabilityZone", "InstanceType"])
new_df = new_df.reindex(df1.columns, axis=1)
new_df = new_df.set_index(df1.index)
pd.testing.assert_frame_equal(df1, new_df)
def test_acquire(self):
with mock.patch.object(SourceProxy.SourceProxy, "__init__",
lambda x, y: None):
aws_s_p = AWSSpotPrice.AWSSpotPrice(config)
with mock.patch.object(SourceProxy.SourceProxy,
'acquire') as acquire:
acquire.return_value = account
with mock.patch.object(boto3.session, 'Session') as s:
s.return_value = SessionMock()
with mock.patch.object(AWSSpotPrice.AWSSpotPriceForRegion,
'get_price') as get_price:
sp_d = utils.input_from_file(
os.path.join(DATA_DIR, 'spot_price.fixture'))
get_price.return_value = sp_d
res = aws_s_p.acquire()
assert produces.keys() == res.keys()
new_df = fix_spot_price(
res['provisioner_resource_spot_prices'])
expected_pandas_df2 = expected_pandas_df.astype(
'object')
pd.testing.assert_frame_equal(expected_pandas_df2,
new_df)
def test_acquire(self):
s = slots.StartdManifests(CONFIG)
with mock.patch.object(htcondor_query.CondorStatus, 'fetch') as f:
f.return_value = utils.input_from_file(FIXTURE_FILE)
pprint.pprint(s.acquire())
def test_acquire(self):
fc = factory_client.FactoryClientManifests(CONFIG)
with mock.patch.object(htcondor_query.CondorStatus, 'fetch') as f:
f.return_value = utils.input_from_file(FIXTURE_FILE)
pprint.pprint(fc.acquire())
def test_acquire(self):
fg = factory_global.FactoryGlobalManifests(CONFIG)
with mock.patch.object(htcondor_query.CondorStatus, "fetch") as f:
f.return_value = utils.input_from_file(FIXTURE_FILE)
pprint.pprint(fg.acquire())
def test_acquire(self):
entries = factory_entries.FactoryEntries(CONFIG_FACTORY_ENTRIES)
with mock.patch.object(htcondor_query.CondorStatus, 'fetch') as f:
f.return_value = utils.input_from_file(FIXTURE_FILE)
pprint.pprint(entries.acquire())
def test_condorq(self):
jq = job_q.JobQ(CONFIG_CQ)
with mock.patch.object(htcondor_query.CondorQ, "fetch") as f:
f.return_value = utils.input_from_file(FIXTURE_FILE)
pprint.pprint(jq.acquire())
def side_effect_get_usage(username):
if username == FAKE_USER:
return {"items": []}
return utils.input_from_file(ALLOCATIONS_FIXTURE_FILE)
