def test_cluster_with_no_fields(self):
# this shouldn't happen in practice; just a robustness check
cluster = MockEmrObject()
summary = _cluster_to_full_summary(cluster)
self.assertEqual(
summary, {
'created': None,
'end': None,
'id': None,
'label': None,
'name': None,
'nih': 0.0,
'nih_bbnu': 0.0,
'nih_billed': 0.0,
'nih_used': 0.0,
'num_steps': 0,
'owner': None,
'pool': None,
'ran': timedelta(0),
'ready': None,
'state': None,
'usage': [],
})
def test_cluster_with_no_steps_split_over_midnight(self):
cluster = dict(
BootstrapActions=[],
Id='j-ISFORJOY',
Name='mr_exciting.woo.20100605.232950.000000',
NormalizedInstanceHours=20,
Status=dict(
State='TERMINATED',
Timeline=dict(
CreationDateTime=parse('2010-06-05T23:30:00Z'),
EndDateTime=parse('2010-06-06T01:15:00Z'),
ReadyDateTime=parse('2010-06-05T23:45:00Z'),
),
),
Steps=[],
)
summary = _cluster_to_full_summary(cluster)
self.assertEqual(summary, {
'created': utc(2010, 6, 5, 23, 30),
'end': utc(2010, 6, 6, 1, 15),
'id': u'j-ISFORJOY',
'label': u'mr_exciting',
'name': u'mr_exciting.woo.20100605.232950.000000',
'nih': 20.0,
'nih_bbnu': 15.0,
'nih_billed': 17.5,
'nih_used': 2.5,
'num_steps': 0,
'owner': u'woo',
'pool': None,
'ran': timedelta(hours=1, minutes=45),
'ready': utc(2010, 6, 5, 23, 45),
'state': u'TERMINATED',
'usage': [{
'date_to_nih_bbnu': {date(2010, 6, 5): 2.5,
date(2010, 6, 6): 12.5},
'date_to_nih_billed': {date(2010, 6, 5): 5.0,
date(2010, 6, 6): 12.5},
'date_to_nih_used': {date(2010, 6, 5): 2.5},
'end': utc(2010, 6, 5, 23, 45),
'end_billing': utc(2010, 6, 6, 1, 15),
'hour_to_nih_bbnu': {utc(2010, 6, 5, 23): 2.5,
utc(2010, 6, 6, 0): 10.0,
utc(2010, 6, 6, 1): 2.5},
'hour_to_nih_billed': {utc(2010, 6, 5, 23): 5.0,
utc(2010, 6, 6, 0): 10.0,
utc(2010, 6, 6, 1): 2.5},
'hour_to_nih_used': {utc(2010, 6, 5, 23): 2.5},
'label': u'mr_exciting',
'nih_bbnu': 15.0,
'nih_billed': 17.5,
'nih_used': 2.5,
'owner': u'woo',
'start': utc(2010, 6, 5, 23, 30),
'step_num': None,
}],
})
def test_cluster_with_no_steps_split_over_midnight(self):
cluster = dict(
BootstrapActions=[],
Id='j-ISFORJOY',
Name='mr_exciting.woo.20100605.232950.000000',
NormalizedInstanceHours=20,
Status=dict(
State='TERMINATED',
Timeline=dict(
CreationDateTime=parse('2010-06-05T23:30:00Z'),
EndDateTime=parse('2010-06-06T01:15:00Z'),
ReadyDateTime=parse('2010-06-05T23:45:00Z'),
),
),
Steps=[],
)
summary = _cluster_to_full_summary(cluster)
self.assertEqual(summary, {
'created': utc(2010, 6, 5, 23, 30),
'end': utc(2010, 6, 6, 1, 15),
'id': u'j-ISFORJOY',
'label': u'mr_exciting',
'name': u'mr_exciting.woo.20100605.232950.000000',
'nih': 20.0,
'nih_bbnu': 15.0,
'nih_billed': 17.5,
'nih_used': 2.5,
'num_steps': 0,
'owner': u'woo',
'pool': None,
'ran': timedelta(hours=1, minutes=45),
'ready': utc(2010, 6, 5, 23, 45),
'state': u'TERMINATED',
'usage': [{
'date_to_nih_bbnu': {date(2010, 6, 5): 2.5,
date(2010, 6, 6): 12.5},
'date_to_nih_billed': {date(2010, 6, 5): 5.0,
date(2010, 6, 6): 12.5},
'date_to_nih_used': {date(2010, 6, 5): 2.5},
'end': utc(2010, 6, 5, 23, 45),
'end_billing': utc(2010, 6, 6, 1, 15),
'hour_to_nih_bbnu': {utc(2010, 6, 5, 23): 2.5,
utc(2010, 6, 6, 0): 10.0,
utc(2010, 6, 6, 1): 2.5},
'hour_to_nih_billed': {utc(2010, 6, 5, 23): 5.0,
utc(2010, 6, 6, 0): 10.0,
utc(2010, 6, 6, 1): 2.5},
'hour_to_nih_used': {utc(2010, 6, 5, 23): 2.5},
'label': u'mr_exciting',
'nih_bbnu': 15.0,
'nih_billed': 17.5,
'nih_used': 2.5,
'owner': u'woo',
'start': utc(2010, 6, 5, 23, 30),
'step_num': None,
}],
})
def test_cluster_with_no_steps_split_over_midnight(self):
cluster = MockEmrObject(
id='j-ISFORJOY',
name='mr_exciting.woo.20100605.232950.000000',
normalizedinstancehours='20',
status=MockEmrObject(
state='TERMINATED',
timeline=MockEmrObject(
creationdatetime='2010-06-05T23:30:00Z',
enddatetime='2010-06-06T01:15:00Z', # 2 hours billed
readydatetime='2010-06-05T23:45:00Z', # 15 minutes "used"
),
),
)
summary = _cluster_to_full_summary(cluster)
self.assertEqual(summary, {
'created': datetime(2010, 6, 5, 23, 30),
'end': datetime(2010, 6, 6, 1, 15),
'id': u'j-ISFORJOY',
'label': u'mr_exciting',
'name': u'mr_exciting.woo.20100605.232950.000000',
'nih': 20.0,
'nih_bbnu': 17.5,
'nih_billed': 20.0,
'nih_used': 2.5,
'num_steps': 0,
'owner': u'woo',
'pool': None,
'ran': timedelta(hours=1, minutes=45),
'ready': datetime(2010, 6, 5, 23, 45),
'state': u'TERMINATED',
'usage': [{
'date_to_nih_bbnu': {date(2010, 6, 5): 2.5,
date(2010, 6, 6): 15.0},
'date_to_nih_billed': {date(2010, 6, 5): 5.0,
date(2010, 6, 6): 15.0},
'date_to_nih_used': {date(2010, 6, 5): 2.5},
'end': datetime(2010, 6, 5, 23, 45),
'end_billing': datetime(2010, 6, 6, 1, 30),
'hour_to_nih_bbnu': {datetime(2010, 6, 5, 23): 2.5,
datetime(2010, 6, 6, 0): 10.0,
datetime(2010, 6, 6, 1): 5.0},
'hour_to_nih_billed': {datetime(2010, 6, 5, 23): 5.0,
datetime(2010, 6, 6, 0): 10.0,
datetime(2010, 6, 6, 1): 5.0},
'hour_to_nih_used': {datetime(2010, 6, 5, 23): 2.5},
'label': u'mr_exciting',
'nih_bbnu': 17.5,
'nih_billed': 20.0,
'nih_used': 2.5,
'owner': u'woo',
'start': datetime(2010, 6, 5, 23, 30),
'step_num': None,
}],
})
def test_still_running_cluster_with_no_steps(self):
cluster = dict(
BootstrapActions=[],
Id='j-ISFORJUICE',
Name='mr_exciting.woo.20100605.235850.000000',
NormalizedInstanceHours=10,
Steps=[],
Status=dict(
State='WAITING',
Timeline=dict(
CreationDateTime=parse('2010-06-06T00:00:00Z'),
ReadyDateTime=parse('2010-06-06T00:15:00Z'),
),
),
)
summary = _cluster_to_full_summary(
cluster, now=utc(2010, 6, 6, 0, 30))
self.assertEqual(summary, {
'created': utc(2010, 6, 6, 0, 0),
'end': None,
'id': u'j-ISFORJUICE',
'label': u'mr_exciting',
'name': u'mr_exciting.woo.20100605.235850.000000',
'nih': 10.0,
'nih_bbnu': 2.5,
'nih_billed': 5.0,
'nih_used': 2.5,
'num_steps': 0,
'owner': u'woo',
'pool': None,
'ran': timedelta(minutes=30),
'ready': utc(2010, 6, 6, 0, 15),
'state': u'WAITING',
'usage': [{
'date_to_nih_bbnu': {date(2010, 6, 6): 2.5},
'date_to_nih_billed': {date(2010, 6, 6): 5.0},
'date_to_nih_used': {date(2010, 6, 6): 2.5},
'end': utc(2010, 6, 6, 0, 15),
'end_billing': utc(2010, 6, 6, 0, 30),
'hour_to_nih_bbnu': {utc(2010, 6, 6, 0): 2.5},
'hour_to_nih_billed': {utc(2010, 6, 6, 0): 5.0},
'hour_to_nih_used': {utc(2010, 6, 6, 0): 2.5},
'label': u'mr_exciting',
'nih_bbnu': 2.5,
'nih_billed': 5.0,
'nih_used': 2.5,
'owner': u'woo',
'start': utc(2010, 6, 6, 0, 0),
'step_num': None,
}],
})
def test_still_running_cluster_with_no_steps(self):
cluster = dict(
BootstrapActions=[],
Id='j-ISFORJUICE',
Name='mr_exciting.woo.20100605.235850.000000',
NormalizedInstanceHours=10,
Steps=[],
Status=dict(
State='WAITING',
Timeline=dict(
CreationDateTime=parse('2010-06-06T00:00:00Z'),
ReadyDateTime=parse('2010-06-06T00:15:00Z'),
),
),
)
summary = _cluster_to_full_summary(
cluster, now=utc(2010, 6, 6, 0, 30))
self.assertEqual(summary, {
'created': utc(2010, 6, 6, 0, 0),
'end': None,
'id': u'j-ISFORJUICE',
'label': u'mr_exciting',
'name': u'mr_exciting.woo.20100605.235850.000000',
'nih': 10.0,
'nih_bbnu': 2.5,
'nih_billed': 5.0,
'nih_used': 2.5,
'num_steps': 0,
'owner': u'woo',
'pool': None,
'ran': timedelta(minutes=30),
'ready': utc(2010, 6, 6, 0, 15),
'state': u'WAITING',
'usage': [{
'date_to_nih_bbnu': {date(2010, 6, 6): 2.5},
'date_to_nih_billed': {date(2010, 6, 6): 5.0},
'date_to_nih_used': {date(2010, 6, 6): 2.5},
'end': utc(2010, 6, 6, 0, 15),
'end_billing': utc(2010, 6, 6, 0, 30),
'hour_to_nih_bbnu': {utc(2010, 6, 6, 0): 2.5},
'hour_to_nih_billed': {utc(2010, 6, 6, 0): 5.0},
'hour_to_nih_used': {utc(2010, 6, 6, 0): 2.5},
'label': u'mr_exciting',
'nih_bbnu': 2.5,
'nih_billed': 5.0,
'nih_used': 2.5,
'owner': u'woo',
'start': utc(2010, 6, 6, 0, 0),
'step_num': None,
}],
})
def test_cluster_that_was_terminated_before_ready(self):
cluster = dict(
BootstrapActions=[],
Id='j-ISFORJOURNEY',
Name='mr_exciting.woo.20100605.235850.000000',
NormalizedInstanceHours=1,
Steps=[],
Status=dict(
State='TERMINATED',
Timeline=dict(
CreationDateTime=parse('2010-06-06T00:00:00Z'),
EndDateTime=parse('2010-06-06T00:30:00Z'),
),
),
)
summary = _cluster_to_full_summary(
cluster, now=utc(2010, 6, 6, 1))
self.assertEqual(summary, {
'created': utc(2010, 6, 6, 0, 0),
'end': utc(2010, 6, 6, 0, 30),
'id': u'j-ISFORJOURNEY',
'label': u'mr_exciting',
'name': u'mr_exciting.woo.20100605.235850.000000',
'nih': 1.0,
'nih_bbnu': 0.0,
'nih_billed': 0.5,
'nih_used': 0.5,
'num_steps': 0,
'owner': u'woo',
'pool': None,
'ran': timedelta(minutes=30),
'ready': None,
'state': u'TERMINATED',
'usage': [{
'date_to_nih_bbnu': {},
'date_to_nih_billed': {date(2010, 6, 6): 0.5},
'date_to_nih_used': {date(2010, 6, 6): 0.5},
'end': utc(2010, 6, 6, 0, 30),
'end_billing': utc(2010, 6, 6, 0, 30),
'hour_to_nih_bbnu': {},
'hour_to_nih_billed': {utc(2010, 6, 6, 0, 0): 0.5},
'hour_to_nih_used': {utc(2010, 6, 6, 0, 0): 0.5},
'label': u'mr_exciting',
'nih_bbnu': 0.0,
'nih_billed': 0.5,
'nih_used': 0.5,
'owner': u'woo',
'start': utc(2010, 6, 6, 0, 0),
'step_num': None,
}],
})
def test_cluster_that_was_terminated_before_ready(self):
cluster = dict(
BootstrapActions=[],
Id='j-ISFORJOURNEY',
Name='mr_exciting.woo.20100605.235850.000000',
NormalizedInstanceHours=1,
Steps=[],
Status=dict(
State='TERMINATED',
Timeline=dict(
CreationDateTime=parse('2010-06-06T00:00:00Z'),
EndDateTime=parse('2010-06-06T00:30:00Z'),
),
),
)
summary = _cluster_to_full_summary(
cluster, now=utc(2010, 6, 6, 1))
self.assertEqual(summary, {
'created': utc(2010, 6, 6, 0, 0),
'end': utc(2010, 6, 6, 0, 30),
'id': u'j-ISFORJOURNEY',
'label': u'mr_exciting',
'name': u'mr_exciting.woo.20100605.235850.000000',
'nih': 1.0,
'nih_bbnu': 0.0,
'nih_billed': 0.5,
'nih_used': 0.5,
'num_steps': 0,
'owner': u'woo',
'pool': None,
'ran': timedelta(minutes=30),
'ready': None,
'state': u'TERMINATED',
'usage': [{
'date_to_nih_bbnu': {},
'date_to_nih_billed': {date(2010, 6, 6): 0.5},
'date_to_nih_used': {date(2010, 6, 6): 0.5},
'end': utc(2010, 6, 6, 0, 30),
'end_billing': utc(2010, 6, 6, 0, 30),
'hour_to_nih_bbnu': {},
'hour_to_nih_billed': {utc(2010, 6, 6, 0, 0): 0.5},
'hour_to_nih_used': {utc(2010, 6, 6, 0, 0): 0.5},
'label': u'mr_exciting',
'nih_bbnu': 0.0,
'nih_billed': 0.5,
'nih_used': 0.5,
'owner': u'woo',
'start': utc(2010, 6, 6, 0, 0),
'step_num': None,
}],
})
def test_still_running_cluster_with_no_steps(self):
cluster = MockEmrObject(
id='j-ISFORJUICE',
name='mr_exciting.woo.20100605.235850.000000',
normalizedinstancehours='10',
status=MockEmrObject(
state='WAITING',
timeline=MockEmrObject(
creationdatetime='2010-06-06T00:00:00Z',
readydatetime='2010-06-06T00:15:00Z',
),
),
)
summary = _cluster_to_full_summary(
cluster, now=datetime(2010, 6, 6, 0, 30))
self.assertEqual(summary, {
'created': datetime(2010, 6, 6, 0, 0),
'end': None,
'id': u'j-ISFORJUICE',
'label': u'mr_exciting',
'name': u'mr_exciting.woo.20100605.235850.000000',
'nih': 10.0,
'nih_bbnu': 2.5,
'nih_billed': 5.0,
'nih_used': 2.5,
'num_steps': 0,
'owner': u'woo',
'pool': None,
'ran': timedelta(minutes=30),
'ready': datetime(2010, 6, 6, 0, 15),
'state': u'WAITING',
'usage': [{
'date_to_nih_bbnu': {date(2010, 6, 6): 2.5},
'date_to_nih_billed': {date(2010, 6, 6): 5.0},
'date_to_nih_used': {date(2010, 6, 6): 2.5},
'end': datetime(2010, 6, 6, 0, 15),
'end_billing': datetime(2010, 6, 6, 0, 30),
'hour_to_nih_bbnu': {datetime(2010, 6, 6, 0): 2.5},
'hour_to_nih_billed': {datetime(2010, 6, 6, 0): 5.0},
'hour_to_nih_used': {datetime(2010, 6, 6, 0): 2.5},
'label': u'mr_exciting',
'nih_bbnu': 2.5,
'nih_billed': 5.0,
'nih_used': 2.5,
'owner': u'woo',
'start': datetime(2010, 6, 6, 0, 0),
'step_num': None,
}],
})
def test_cluster_that_was_terminated_before_ready(self):
cluster = MockEmrObject(
id='j-ISFORJOURNEY',
name='mr_exciting.woo.20100605.235850.000000',
normalizedinstancehours='1',
status=MockEmrObject(
state='TERMINATED',
timeline=MockEmrObject(
creationdatetime='2010-06-06T00:00:00Z',
enddatetime='2010-06-06T00:30:00Z',
),
),
)
summary = _cluster_to_full_summary(
cluster, now=datetime(2010, 6, 6, 1))
self.assertEqual(summary, {
'created': datetime(2010, 6, 6, 0, 0),
'end': datetime(2010, 6, 6, 0, 30),
'id': u'j-ISFORJOURNEY',
'label': u'mr_exciting',
'name': u'mr_exciting.woo.20100605.235850.000000',
'nih': 1.0,
'nih_bbnu': 0.5,
'nih_billed': 1.0,
'nih_used': 0.5,
'num_steps': 0,
'owner': u'woo',
'pool': None,
'ran': timedelta(minutes=30),
'ready': None,
'state': u'TERMINATED',
'usage': [{
'date_to_nih_bbnu': {date(2010, 6, 6): 0.5},
'date_to_nih_billed': {date(2010, 6, 6): 1.0},
'date_to_nih_used': {date(2010, 6, 6): 0.5},
'end': datetime(2010, 6, 6, 0, 30),
'end_billing': datetime(2010, 6, 6, 1, 0),
'hour_to_nih_bbnu': {datetime(2010, 6, 6, 0, 0): 0.5},
'hour_to_nih_billed': {datetime(2010, 6, 6, 0, 0): 1.0},
'hour_to_nih_used': {datetime(2010, 6, 6, 0, 0): 0.5},
'label': u'mr_exciting',
'nih_bbnu': 0.5,
'nih_billed': 1.0,
'nih_used': 0.5,
'owner': u'woo',
'start': datetime(2010, 6, 6, 0, 0),
'step_num': None,
}],
})
def test_cluster_with_no_fields(self):
# this shouldn't happen in practice; just a robustness check
cluster = MockEmrObject()
summary = _cluster_to_full_summary(cluster)
self.assertEqual(summary, {
'created': None,
'end': None,
'id': None,
'label': None,
'name': None,
'nih': 0.0,
'nih_bbnu': 0.0,
'nih_billed': 0.0,
'nih_used': 0.0,
'num_steps': 0,
'owner': None,
'pool': None,
'ran': timedelta(0),
'ready': None,
'state': None,
'usage': [],
})
def test_multi_step_cluster(self):
cluster = dict(
BootstrapActions=[],
Id='j-ISFORJOB',
Name='mr_exciting.woo.20100605.232850.000000',
NormalizedInstanceHours=20,
Status=dict(
State='TERMINATED',
Timeline=dict(
CreationDateTime=parse('2010-06-05T23:30:00Z'),
EndDateTime=parse(
'2010-06-06T01:15:00Z'), # 2 hours are billed
ReadyDateTime=parse('2010-06-05T23:45:00Z'),
),
),
Steps=[
dict(
Name='mr_exciting.woo.20100605.232850.000000: Step 1 of 3',
Status=dict(Timeline=dict(
StartDateTime=parse('2010-06-05T23:45:00Z'),
EndDateTime=parse('2010-06-06T00:15:00Z'),
), ),
),
dict(
Name='mr_exciting.woo.20100605.232850.000000: Step 2 of 3',
Status=dict(Timeline=dict(
StartDateTime=parse('2010-06-06T00:30:00Z'),
EndDateTime=parse('2010-06-06T00:45:00Z'),
), ),
),
dict(
Name='mr_exciting.woo.20100605.232850.000000: Step 3 of 3',
Status=dict(Timeline=dict(
StartDateTime=parse('2010-06-06T00:45:00Z'),
EndDateTime=parse('2010-06-06T01:00:00Z'),
), ),
),
],
)
summary = _cluster_to_full_summary(cluster)
self.assertEqual(
summary,
{
'created':
utc(2010, 6, 5, 23, 30),
'end':
utc(2010, 6, 6, 1, 15),
'id':
u'j-ISFORJOB',
'label':
u'mr_exciting',
'name':
u'mr_exciting.woo.20100605.232850.000000',
'nih':
20.0,
'nih_bbnu':
7.5,
'nih_billed':
20.0,
'nih_used':
12.5,
'num_steps':
3,
'owner':
u'woo',
'pool':
None,
'ran':
timedelta(hours=1, minutes=45),
'ready':
utc(2010, 6, 5, 23, 45),
'state':
u'TERMINATED',
'usage': [
{
# bootstrapping
'date_to_nih_used': {
date(2010, 6, 5): 2.5
},
'date_to_nih_bbnu': {},
'date_to_nih_billed': {
date(2010, 6, 5): 2.5
},
'end': utc(2010, 6, 5, 23, 45),
'end_billing': utc(2010, 6, 5, 23, 45),
'hour_to_nih_bbnu': {},
'hour_to_nih_billed': {
utc(2010, 6, 5, 23): 2.5
},
'hour_to_nih_used': {
utc(2010, 6, 5, 23): 2.5
},
'label': u'mr_exciting',
'nih_used': 2.5,
'nih_bbnu': 0.0,
'nih_billed': 2.5,
'owner': u'woo',
'start': utc(2010, 6, 5, 23, 30),
'step_num': None,
},
{
# step 1 (and idle time after)
'date_to_nih_used': {
date(2010, 6, 5): 2.5,
date(2010, 6, 6): 2.5
},
'date_to_nih_bbnu': {
date(2010, 6, 6): 2.5
},
'date_to_nih_billed': {
date(2010, 6, 5): 2.5,
date(2010, 6, 6): 5.0
},
'end': utc(2010, 6, 6, 0, 15),
'end_billing': utc(2010, 6, 6, 0, 30),
'hour_to_nih_bbnu': {
utc(2010, 6, 6, 0): 2.5
},
'hour_to_nih_billed': {
utc(2010, 6, 5, 23): 2.5,
utc(2010, 6, 6, 0): 5.0
},
'hour_to_nih_used': {
utc(2010, 6, 5, 23): 2.5,
utc(2010, 6, 6, 0): 2.5
},
'label': u'mr_exciting',
'nih_used': 5.0,
'nih_bbnu': 2.5,
'nih_billed': 7.5,
'owner': u'woo',
'start': utc(2010, 6, 5, 23, 45),
'step_num': 1,
},
{
# step 2
'date_to_nih_used': {
date(2010, 6, 6): 2.5
},
'date_to_nih_bbnu': {},
'date_to_nih_billed': {
date(2010, 6, 6): 2.5
},
'end': utc(2010, 6, 6, 0, 45),
'end_billing': utc(2010, 6, 6, 0, 45),
'hour_to_nih_bbnu': {},
'hour_to_nih_billed': {
utc(2010, 6, 6, 0): 2.5
},
'hour_to_nih_used': {
utc(2010, 6, 6, 0): 2.5
},
'label': u'mr_exciting',
'nih_used': 2.5,
'nih_bbnu': 0.0,
'nih_billed': 2.5,
'owner': u'woo',
'start': utc(2010, 6, 6, 0, 30),
'step_num': 2,
},
{
# step 3 (and idle time after)
'date_to_nih_used': {
date(2010, 6, 6): 2.5
},
'date_to_nih_bbnu': {
date(2010, 6, 6): 5.0
},
'date_to_nih_billed': {
date(2010, 6, 6): 7.5
},
'end': utc(2010, 6, 6, 1, 0),
'end_billing': utc(2010, 6, 6, 1, 30),
'hour_to_nih_bbnu': {
utc(2010, 6, 6, 1): 5.0
},
'hour_to_nih_billed': {
utc(2010, 6, 6, 0): 2.5,
utc(2010, 6, 6, 1): 5.0
},
'hour_to_nih_used': {
utc(2010, 6, 6, 0): 2.5
},
'label': u'mr_exciting',
'nih_used': 2.5,
'nih_bbnu': 5.0,
'nih_billed': 7.5,
'owner': u'woo',
'start': utc(2010, 6, 6, 0, 45),
'step_num': 3,
}
],
})
def test_pooled_cluster(self):
# same as test case above with different job keys
cluster = MockEmrObject(
bootstrapactions=[
MockEmrObject(args=[]),
MockEmrObject(args=[
MockEmrObject(
value='pool-0123456789abcdef0123456789abcdef'),
MockEmrObject(value='reflecting'),
]),
],
id='j-ISFORJOB',
name='mr_exciting.woo.20100605.232850.000000',
normalizedinstancehours='20',
state='TERMINATED',
status=MockEmrObject(
state='TERMINATED',
timeline=MockEmrObject(
creationdatetime='2010-06-05T23:30:00Z',
enddatetime='2010-06-06T01:15:00Z', # 2 hours are billed
readydatetime='2010-06-05T23:45:00Z',
),
),
steps=[
MockEmrObject(
name='mr_exciting.woo.20100605.232950.000000: Step 1 of 1',
status=MockEmrObject(timeline=MockEmrObject(
startdatetime='2010-06-05T23:45:00Z',
enddatetime='2010-06-06T00:15:00Z',
), ),
),
MockEmrObject(
name='mr_whatever.meh.20100606.002000.000000: Step 1 of 2',
status=MockEmrObject(timeline=MockEmrObject(
startdatetime='2010-06-06T00:30:00Z',
enddatetime='2010-06-06T00:45:00Z',
), ),
),
MockEmrObject(
name='mr_whatever.meh.20100606.002000.000000: Step 2 of 2',
status=MockEmrObject(timeline=MockEmrObject(
startdatetime='2010-06-06T00:45:00Z',
enddatetime='2010-06-06T01:00:00Z',
), ),
),
],
)
summary = _cluster_to_full_summary(cluster)
self.assertEqual(
summary,
{
'created':
datetime(2010, 6, 5, 23, 30),
'end':
datetime(2010, 6, 6, 1, 15),
'id':
u'j-ISFORJOB',
'label':
u'mr_exciting',
'name':
u'mr_exciting.woo.20100605.232850.000000',
'nih':
20.0,
'nih_bbnu':
7.5,
'nih_billed':
20.0,
'nih_used':
12.5,
'num_steps':
3,
'owner':
u'woo',
'pool':
u'reflecting',
'ran':
timedelta(hours=1, minutes=45),
'ready':
datetime(2010, 6, 5, 23, 45),
'state':
'TERMINATED',
'usage': [
{
# bootstrapping
'date_to_nih_used': {
date(2010, 6, 5): 2.5
},
'date_to_nih_bbnu': {},
'date_to_nih_billed': {
date(2010, 6, 5): 2.5
},
'end': datetime(2010, 6, 5, 23, 45),
'end_billing': datetime(2010, 6, 5, 23, 45),
'hour_to_nih_bbnu': {},
'hour_to_nih_billed': {
datetime(2010, 6, 5, 23): 2.5
},
'hour_to_nih_used': {
datetime(2010, 6, 5, 23): 2.5
},
'label': u'mr_exciting',
'nih_used': 2.5,
'nih_bbnu': 0.0,
'nih_billed': 2.5,
'owner': u'woo',
'start': datetime(2010, 6, 5, 23, 30),
'step_num': None,
},
{
# mr_exciting, step 1 (and idle time after)
'date_to_nih_used': {
date(2010, 6, 5): 2.5,
date(2010, 6, 6): 2.5
},
'date_to_nih_bbnu': {
date(2010, 6, 6): 2.5
},
'date_to_nih_billed': {
date(2010, 6, 5): 2.5,
date(2010, 6, 6): 5.0
},
'end': datetime(2010, 6, 6, 0, 15),
'end_billing': datetime(2010, 6, 6, 0, 30),
'hour_to_nih_bbnu': {
datetime(2010, 6, 6, 0): 2.5
},
'hour_to_nih_billed': {
datetime(2010, 6, 5, 23): 2.5,
datetime(2010, 6, 6, 0): 5.0
},
'hour_to_nih_used': {
datetime(2010, 6, 5, 23): 2.5,
datetime(2010, 6, 6, 0): 2.5
},
'label': u'mr_exciting',
'nih_used': 5.0,
'nih_bbnu': 2.5,
'nih_billed': 7.5,
'owner': u'woo',
'start': datetime(2010, 6, 5, 23, 45),
'step_num': 1,
},
{
# mr whatever, step 1
'date_to_nih_used': {
date(2010, 6, 6): 2.5
},
'date_to_nih_bbnu': {},
'date_to_nih_billed': {
date(2010, 6, 6): 2.5
},
'end': datetime(2010, 6, 6, 0, 45),
'end_billing': datetime(2010, 6, 6, 0, 45),
'hour_to_nih_bbnu': {},
'hour_to_nih_billed': {
datetime(2010, 6, 6, 0): 2.5
},
'hour_to_nih_used': {
datetime(2010, 6, 6, 0): 2.5
},
'label': u'mr_whatever',
'nih_used': 2.5,
'nih_bbnu': 0.0,
'nih_billed': 2.5,
'owner': u'meh',
'start': datetime(2010, 6, 6, 0, 30),
'step_num': 1,
},
# mr whatever, step 2 (and idle time after)
{
'date_to_nih_used': {
date(2010, 6, 6): 2.5
},
'date_to_nih_bbnu': {
date(2010, 6, 6): 5.0
},
'date_to_nih_billed': {
date(2010, 6, 6): 7.5
},
'end': datetime(2010, 6, 6, 1, 0),
'end_billing': datetime(2010, 6, 6, 1, 30),
'hour_to_nih_bbnu': {
datetime(2010, 6, 6, 1): 5.0
},
'hour_to_nih_billed': {
datetime(2010, 6, 6, 0): 2.5,
datetime(2010, 6, 6, 1): 5.0
},
'hour_to_nih_used': {
datetime(2010, 6, 6, 0): 2.5
},
'label': u'mr_whatever',
'nih_used': 2.5,
'nih_bbnu': 5.0,
'nih_billed': 7.5,
'owner': u'meh',
'start': datetime(2010, 6, 6, 0, 45),
'step_num': 2,
}
],
})
def test_multi_step_cluster(self):
cluster = MockEmrObject(
id='j-ISFORJOB',
name='mr_exciting.woo.20100605.232850.000000',
normalizedinstancehours='20',
status=MockEmrObject(
state='TERMINATED',
timeline=MockEmrObject(
creationdatetime='2010-06-05T23:30:00Z',
enddatetime='2010-06-06T01:15:00Z', # 2 hours are billed
readydatetime='2010-06-05T23:45:00Z',
),
),
steps=[
MockEmrObject(
name='mr_exciting.woo.20100605.232850.000000: Step 1 of 3',
status=MockEmrObject(timeline=MockEmrObject(
startdatetime='2010-06-05T23:45:00Z',
enddatetime='2010-06-06T00:15:00Z',
), ),
),
MockEmrObject(
name='mr_exciting.woo.20100605.232850.000000: Step 2 of 3',
status=MockEmrObject(timeline=MockEmrObject(
startdatetime='2010-06-06T00:30:00Z',
enddatetime='2010-06-06T00:45:00Z',
), ),
),
MockEmrObject(
name='mr_exciting.woo.20100605.232850.000000: Step 3 of 3',
status=MockEmrObject(timeline=MockEmrObject(
startdatetime='2010-06-06T00:45:00Z',
enddatetime='2010-06-06T01:00:00Z',
), ),
),
],
)
summary = _cluster_to_full_summary(cluster)
self.assertEqual(
summary,
{
'created':
datetime(2010, 6, 5, 23, 30),
'end':
datetime(2010, 6, 6, 1, 15),
'id':
u'j-ISFORJOB',
'label':
u'mr_exciting',
'name':
u'mr_exciting.woo.20100605.232850.000000',
'nih':
20.0,
'nih_bbnu':
7.5,
'nih_billed':
20.0,
'nih_used':
12.5,
'num_steps':
3,
'owner':
u'woo',
'pool':
None,
'ran':
timedelta(hours=1, minutes=45),
'ready':
datetime(2010, 6, 5, 23, 45),
'state':
u'TERMINATED',
'usage': [
{
# bootstrapping
'date_to_nih_used': {
date(2010, 6, 5): 2.5
},
'date_to_nih_bbnu': {},
'date_to_nih_billed': {
date(2010, 6, 5): 2.5
},
'end': datetime(2010, 6, 5, 23, 45),
'end_billing': datetime(2010, 6, 5, 23, 45),
'hour_to_nih_bbnu': {},
'hour_to_nih_billed': {
datetime(2010, 6, 5, 23): 2.5
},
'hour_to_nih_used': {
datetime(2010, 6, 5, 23): 2.5
},
'label': u'mr_exciting',
'nih_used': 2.5,
'nih_bbnu': 0.0,
'nih_billed': 2.5,
'owner': u'woo',
'start': datetime(2010, 6, 5, 23, 30),
'step_num': None,
},
{
# step 1 (and idle time after)
'date_to_nih_used': {
date(2010, 6, 5): 2.5,
date(2010, 6, 6): 2.5
},
'date_to_nih_bbnu': {
date(2010, 6, 6): 2.5
},
'date_to_nih_billed': {
date(2010, 6, 5): 2.5,
date(2010, 6, 6): 5.0
},
'end': datetime(2010, 6, 6, 0, 15),
'end_billing': datetime(2010, 6, 6, 0, 30),
'hour_to_nih_bbnu': {
datetime(2010, 6, 6, 0): 2.5
},
'hour_to_nih_billed': {
datetime(2010, 6, 5, 23): 2.5,
datetime(2010, 6, 6, 0): 5.0
},
'hour_to_nih_used': {
datetime(2010, 6, 5, 23): 2.5,
datetime(2010, 6, 6, 0): 2.5
},
'label': u'mr_exciting',
'nih_used': 5.0,
'nih_bbnu': 2.5,
'nih_billed': 7.5,
'owner': u'woo',
'start': datetime(2010, 6, 5, 23, 45),
'step_num': 1,
},
{
# step 2
'date_to_nih_used': {
date(2010, 6, 6): 2.5
},
'date_to_nih_bbnu': {},
'date_to_nih_billed': {
date(2010, 6, 6): 2.5
},
'end': datetime(2010, 6, 6, 0, 45),
'end_billing': datetime(2010, 6, 6, 0, 45),
'hour_to_nih_bbnu': {},
'hour_to_nih_billed': {
datetime(2010, 6, 6, 0): 2.5
},
'hour_to_nih_used': {
datetime(2010, 6, 6, 0): 2.5
},
'label': u'mr_exciting',
'nih_used': 2.5,
'nih_bbnu': 0.0,
'nih_billed': 2.5,
'owner': u'woo',
'start': datetime(2010, 6, 6, 0, 30),
'step_num': 2,
},
# step 3 (and idle time after)
{
'date_to_nih_used': {
date(2010, 6, 6): 2.5
},
'date_to_nih_bbnu': {
date(2010, 6, 6): 5.0
},
'date_to_nih_billed': {
date(2010, 6, 6): 7.5
},
'end': datetime(2010, 6, 6, 1, 0),
'end_billing': datetime(2010, 6, 6, 1, 30),
'hour_to_nih_bbnu': {
datetime(2010, 6, 6, 1): 5.0
},
'hour_to_nih_billed': {
datetime(2010, 6, 6, 0): 2.5,
datetime(2010, 6, 6, 1): 5.0
},
'hour_to_nih_used': {
datetime(2010, 6, 6, 0): 2.5
},
'label': u'mr_exciting',
'nih_used': 2.5,
'nih_bbnu': 5.0,
'nih_billed': 7.5,
'owner': u'woo',
'start': datetime(2010, 6, 6, 0, 45),
'step_num': 3,
}
],
})
def test_cluster_with_one_cancelled_step(self):
cluster = MockEmrObject(
id='j-ISFORJACUZZI',
name='mr_exciting.woo.20100606.035855.000000',
normalizedinstancehours='20',
status=MockEmrObject(
state='RUNNING',
timeline=MockEmrObject(
creationdatetime='2010-06-06T04:00:00Z',
enddatetime='2010-06-06T05:30:00Z',
readydatetime='2010-06-06T04:15:00Z',
),
),
# step doesn't have end time even though cluster does
steps=[
MockEmrObject(
name='mr_exciting.woo.20100606.035855.000000: Step 1 of 3',
status=MockEmrObject(timeline=MockEmrObject(
startdatetime='2010-06-06T04:15:00Z', ), ),
),
])
summary = _cluster_to_full_summary(cluster)
self.assertEqual(
summary,
{
'created':
datetime(2010, 6, 6, 4, 0),
'end':
datetime(2010, 6, 6, 5, 30),
'id':
u'j-ISFORJACUZZI',
'label':
u'mr_exciting',
'name':
u'mr_exciting.woo.20100606.035855.000000',
'nih':
20.0,
'nih_bbnu':
17.5,
'nih_billed':
20.0,
'nih_used':
2.5,
'num_steps':
1,
'owner':
u'woo',
'pool':
None,
'ran':
timedelta(hours=1, minutes=30),
'ready':
datetime(2010, 6, 6, 4, 15),
'state':
u'RUNNING',
'usage': [
{
# bootstrapping
'date_to_nih_used': {
date(2010, 6, 6): 2.5
},
'date_to_nih_bbnu': {},
'date_to_nih_billed': {
date(2010, 6, 6): 2.5
},
'end': datetime(2010, 6, 6, 4, 15),
'end_billing': datetime(2010, 6, 6, 4, 15),
'hour_to_nih_used': {
datetime(2010, 6, 6, 4): 2.5
},
'hour_to_nih_bbnu': {},
'hour_to_nih_billed': {
datetime(2010, 6, 6, 4): 2.5
},
'label': u'mr_exciting',
'nih_used': 2.5,
'nih_bbnu': 0.0,
'nih_billed': 2.5,
'owner': u'woo',
'start': datetime(2010, 6, 6, 4, 0),
'step_num': None,
},
{
# mr_exciting, step 1 (cancelled)
'date_to_nih_used': {},
'date_to_nih_bbnu': {
date(2010, 6, 6): 17.5
},
'date_to_nih_billed': {
date(2010, 6, 6): 17.5
},
'end': datetime(2010, 6, 6, 4, 15),
'end_billing': datetime(2010, 6, 6, 6, 0),
'hour_to_nih_used': {},
'hour_to_nih_bbnu': {
datetime(2010, 6, 6, 4): 7.5,
datetime(2010, 6, 6, 5): 10.0
},
'hour_to_nih_billed': {
datetime(2010, 6, 6, 4): 7.5,
datetime(2010, 6, 6, 5): 10.0
},
'label': u'mr_exciting',
'nih_used': 0.0,
'nih_bbnu': 17.5,
'nih_billed': 17.5,
'owner': u'woo',
'start': datetime(2010, 6, 6, 4, 15),
'step_num': 1,
}
],
})
def test_basic_cluster_with_no_steps(self):
cluster = MockEmrObject(
id='j-ISFORJAGUAR',
name='mr_exciting.woo.20100605.235850.000000',
normalizedinstancehours='10',
status=MockEmrObject(
state='TERMINATED',
timeline=MockEmrObject(
creationdatetime='2010-06-06T00:00:00Z',
enddatetime='2010-06-06T00:30:00Z',
readydatetime='2010-06-06T00:15:00Z',
),
),
)
summary = _cluster_to_full_summary(cluster)
self.assertEqual(
summary,
{
'created':
datetime(2010, 6, 6, 0, 0),
'end':
datetime(2010, 6, 6, 0, 30),
'id':
u'j-ISFORJAGUAR',
'label':
u'mr_exciting',
'name':
u'mr_exciting.woo.20100605.235850.000000',
'nih':
10.0,
'nih_bbnu':
7.5,
'nih_billed':
10.0,
'nih_used':
2.5, # only a quarter of time billed was used
'num_steps':
0,
'owner':
u'woo',
'pool':
None,
'ran':
timedelta(minutes=30),
'ready':
datetime(2010, 6, 6, 0, 15),
'state':
u'TERMINATED',
'usage': [{
'date_to_nih_bbnu': {
date(2010, 6, 6): 7.5
},
'date_to_nih_billed': {
date(2010, 6, 6): 10.0
},
'date_to_nih_used': {
date(2010, 6, 6): 2.5
},
'end': datetime(2010, 6, 6, 0, 15),
'end_billing': datetime(2010, 6, 6, 1, 0),
'label': u'mr_exciting',
'hour_to_nih_bbnu': {
datetime(2010, 6, 6, 0): 7.5
},
'hour_to_nih_billed': {
datetime(2010, 6, 6, 0): 10.0
},
'hour_to_nih_used': {
datetime(2010, 6, 6, 0): 2.5
},
'nih_bbnu': 7.5,
'nih_billed': 10.0,
'nih_used': 2.5,
'owner': u'woo',
'start': datetime(2010, 6, 6, 0, 0),
'step_num': None,
}],
})
def test_cluster_with_no_steps_split_over_midnight(self):
cluster = MockEmrObject(
id='j-ISFORJOY',
name='mr_exciting.woo.20100605.232950.000000',
normalizedinstancehours='20',
status=MockEmrObject(
state='TERMINATED',
timeline=MockEmrObject(
creationdatetime='2010-06-05T23:30:00Z',
enddatetime='2010-06-06T01:15:00Z', # 2 hours billed
readydatetime='2010-06-05T23:45:00Z', # 15 minutes "used"
),
),
)
summary = _cluster_to_full_summary(cluster)
self.assertEqual(
summary, {
'created':
datetime(2010, 6, 5, 23, 30),
'end':
datetime(2010, 6, 6, 1, 15),
'id':
u'j-ISFORJOY',
'label':
u'mr_exciting',
'name':
u'mr_exciting.woo.20100605.232950.000000',
'nih':
20.0,
'nih_bbnu':
17.5,
'nih_billed':
20.0,
'nih_used':
2.5,
'num_steps':
0,
'owner':
u'woo',
'pool':
None,
'ran':
timedelta(hours=1, minutes=45),
'ready':
datetime(2010, 6, 5, 23, 45),
'state':
u'TERMINATED',
'usage': [{
'date_to_nih_bbnu': {
date(2010, 6, 5): 2.5,
date(2010, 6, 6): 15.0
},
'date_to_nih_billed': {
date(2010, 6, 5): 5.0,
date(2010, 6, 6): 15.0
},
'date_to_nih_used': {
date(2010, 6, 5): 2.5
},
'end': datetime(2010, 6, 5, 23, 45),
'end_billing': datetime(2010, 6, 6, 1, 30),
'hour_to_nih_bbnu': {
datetime(2010, 6, 5, 23): 2.5,
datetime(2010, 6, 6, 0): 10.0,
datetime(2010, 6, 6, 1): 5.0
},
'hour_to_nih_billed': {
datetime(2010, 6, 5, 23): 5.0,
datetime(2010, 6, 6, 0): 10.0,
datetime(2010, 6, 6, 1): 5.0
},
'hour_to_nih_used': {
datetime(2010, 6, 5, 23): 2.5
},
'label': u'mr_exciting',
'nih_bbnu': 17.5,
'nih_billed': 20.0,
'nih_used': 2.5,
'owner': u'woo',
'start': datetime(2010, 6, 5, 23, 30),
'step_num': None,
}],
})
def test_cluster_that_was_terminated_before_ready(self):
cluster = MockEmrObject(
id='j-ISFORJOURNEY',
name='mr_exciting.woo.20100605.235850.000000',
normalizedinstancehours='1',
status=MockEmrObject(
state='TERMINATED',
timeline=MockEmrObject(
creationdatetime='2010-06-06T00:00:00Z',
enddatetime='2010-06-06T00:30:00Z',
),
),
)
summary = _cluster_to_full_summary(cluster,
now=datetime(2010, 6, 6, 1))
self.assertEqual(
summary, {
'created':
datetime(2010, 6, 6, 0, 0),
'end':
datetime(2010, 6, 6, 0, 30),
'id':
u'j-ISFORJOURNEY',
'label':
u'mr_exciting',
'name':
u'mr_exciting.woo.20100605.235850.000000',
'nih':
1.0,
'nih_bbnu':
0.5,
'nih_billed':
1.0,
'nih_used':
0.5,
'num_steps':
0,
'owner':
u'woo',
'pool':
None,
'ran':
timedelta(minutes=30),
'ready':
None,
'state':
u'TERMINATED',
'usage': [{
'date_to_nih_bbnu': {
date(2010, 6, 6): 0.5
},
'date_to_nih_billed': {
date(2010, 6, 6): 1.0
},
'date_to_nih_used': {
date(2010, 6, 6): 0.5
},
'end': datetime(2010, 6, 6, 0, 30),
'end_billing': datetime(2010, 6, 6, 1, 0),
'hour_to_nih_bbnu': {
datetime(2010, 6, 6, 0, 0): 0.5
},
'hour_to_nih_billed': {
datetime(2010, 6, 6, 0, 0): 1.0
},
'hour_to_nih_used': {
datetime(2010, 6, 6, 0, 0): 0.5
},
'label': u'mr_exciting',
'nih_bbnu': 0.5,
'nih_billed': 1.0,
'nih_used': 0.5,
'owner': u'woo',
'start': datetime(2010, 6, 6, 0, 0),
'step_num': None,
}],
})
def test_multi_step_cluster(self):
cluster = dict(
BootstrapActions=[],
Id='j-ISFORJOB',
Name='mr_exciting.woo.20100605.232850.000000',
NormalizedInstanceHours=20,
Status=dict(
State='TERMINATED',
Timeline=dict(
CreationDateTime=parse('2010-06-05T23:30:00Z'),
EndDateTime=parse('2010-06-06T01:15:00Z'),
ReadyDateTime=parse('2010-06-05T23:45:00Z'),
),
),
Steps=[
dict(
Name='mr_exciting.woo.20100605.232850.000000: Step 1 of 3',
Status=dict(
Timeline=dict(
StartDateTime=parse('2010-06-05T23:45:00Z'),
EndDateTime=parse('2010-06-06T00:15:00Z'),
),
),
),
dict(
Name='mr_exciting.woo.20100605.232850.000000: Step 2 of 3',
Status=dict(
Timeline=dict(
StartDateTime=parse('2010-06-06T00:30:00Z'),
EndDateTime=parse('2010-06-06T00:45:00Z'),
),
),
),
dict(
Name='mr_exciting.woo.20100605.232850.000000: Step 3 of 3',
Status=dict(
Timeline=dict(
StartDateTime=parse('2010-06-06T00:45:00Z'),
EndDateTime=parse('2010-06-06T01:00:00Z'),
),
),
),
],
)
summary = _cluster_to_full_summary(cluster)
self.assertEqual(summary, {
'created': utc(2010, 6, 5, 23, 30),
'end': utc(2010, 6, 6, 1, 15),
'id': u'j-ISFORJOB',
'label': u'mr_exciting',
'name': u'mr_exciting.woo.20100605.232850.000000',
'nih': 20.0,
'nih_bbnu': 5.0,
'nih_billed': 17.5,
'nih_used': 12.5,
'num_steps': 3,
'owner': u'woo',
'pool': None,
'ran': timedelta(hours=1, minutes=45),
'ready': utc(2010, 6, 5, 23, 45),
'state': u'TERMINATED',
'usage': [{
# bootstrapping
'date_to_nih_used': {date(2010, 6, 5): 2.5},
'date_to_nih_bbnu': {},
'date_to_nih_billed': {date(2010, 6, 5): 2.5},
'end': utc(2010, 6, 5, 23, 45),
'end_billing': utc(2010, 6, 5, 23, 45),
'hour_to_nih_bbnu': {},
'hour_to_nih_billed': {utc(2010, 6, 5, 23): 2.5},
'hour_to_nih_used': {utc(2010, 6, 5, 23): 2.5},
'label': u'mr_exciting',
'nih_used': 2.5,
'nih_bbnu': 0.0,
'nih_billed': 2.5,
'owner': u'woo',
'start': utc(2010, 6, 5, 23, 30),
'step_num': None,
}, {
# step 1 (and idle time after)
'date_to_nih_used': {date(2010, 6, 5): 2.5,
date(2010, 6, 6): 2.5},
'date_to_nih_bbnu': {date(2010, 6, 6): 2.5},
'date_to_nih_billed': {date(2010, 6, 5): 2.5,
date(2010, 6, 6): 5.0},
'end': utc(2010, 6, 6, 0, 15),
'end_billing': utc(2010, 6, 6, 0, 30),
'hour_to_nih_bbnu': {utc(2010, 6, 6, 0): 2.5},
'hour_to_nih_billed': {utc(2010, 6, 5, 23): 2.5,
utc(2010, 6, 6, 0): 5.0},
'hour_to_nih_used': {utc(2010, 6, 5, 23): 2.5,
utc(2010, 6, 6, 0): 2.5},
'label': u'mr_exciting',
'nih_used': 5.0,
'nih_bbnu': 2.5,
'nih_billed': 7.5,
'owner': u'woo',
'start': utc(2010, 6, 5, 23, 45),
'step_num': 1,
}, {
# step 2
'date_to_nih_used': {date(2010, 6, 6): 2.5},
'date_to_nih_bbnu': {},
'date_to_nih_billed': {date(2010, 6, 6): 2.5},
'end': utc(2010, 6, 6, 0, 45),
'end_billing': utc(2010, 6, 6, 0, 45),
'hour_to_nih_bbnu': {},
'hour_to_nih_billed': {utc(2010, 6, 6, 0): 2.5},
'hour_to_nih_used': {utc(2010, 6, 6, 0): 2.5},
'label': u'mr_exciting',
'nih_used': 2.5,
'nih_bbnu': 0.0,
'nih_billed': 2.5,
'owner': u'woo',
'start': utc(2010, 6, 6, 0, 30),
'step_num': 2,
}, {
# step 3 (and idle time after)
'date_to_nih_used': {date(2010, 6, 6): 2.5},
'date_to_nih_bbnu': {date(2010, 6, 6): 2.5},
'date_to_nih_billed': {date(2010, 6, 6): 5.0},
'end': utc(2010, 6, 6, 1, 0),
'end_billing': utc(2010, 6, 6, 1, 15),
'hour_to_nih_bbnu': {utc(2010, 6, 6, 1): 2.5},
'hour_to_nih_billed': {utc(2010, 6, 6, 0): 2.5,
utc(2010, 6, 6, 1): 2.5},
'hour_to_nih_used': {utc(2010, 6, 6, 0): 2.5},
'label': u'mr_exciting',
'nih_used': 2.5,
'nih_bbnu': 2.5,
'nih_billed': 5.0,
'owner': u'woo',
'start': utc(2010, 6, 6, 0, 45),
'step_num': 3,
}],
})
def test_cluster_with_one_cancelled_step(self):
cluster = dict(
BootstrapActions=[],
Id='j-ISFORJACUZZI',
Name='mr_exciting.woo.20100606.035855.000000',
NormalizedInstanceHours=20,
Status=dict(
State='RUNNING',
Timeline=dict(
CreationDateTime=parse('2010-06-06T04:00:00Z'),
EndDateTime=parse('2010-06-06T05:30:00Z'),
ReadyDateTime=parse('2010-06-06T04:15:00Z'),
),
),
# step doesn't have end time even though cluster does
Steps=[
dict(
Name='mr_exciting.woo.20100606.035855.000000: Step 1 of 3',
Status=dict(Timeline=dict(
StartDateTime=parse('2010-06-06T04:15:00Z'), ), ),
),
])
summary = _cluster_to_full_summary(cluster)
self.assertEqual(
summary,
{
'created':
utc(2010, 6, 6, 4, 0),
'end':
utc(2010, 6, 6, 5, 30),
'id':
u'j-ISFORJACUZZI',
'label':
u'mr_exciting',
'name':
u'mr_exciting.woo.20100606.035855.000000',
'nih':
20.0,
'nih_bbnu':
17.5,
'nih_billed':
20.0,
'nih_used':
2.5,
'num_steps':
1,
'owner':
u'woo',
'pool':
None,
'ran':
timedelta(hours=1, minutes=30),
'ready':
utc(2010, 6, 6, 4, 15),
'state':
u'RUNNING',
'usage': [
{
# bootstrapping
'date_to_nih_used': {
date(2010, 6, 6): 2.5
},
'date_to_nih_bbnu': {},
'date_to_nih_billed': {
date(2010, 6, 6): 2.5
},
'end': utc(2010, 6, 6, 4, 15),
'end_billing': utc(2010, 6, 6, 4, 15),
'hour_to_nih_used': {
utc(2010, 6, 6, 4): 2.5
},
'hour_to_nih_bbnu': {},
'hour_to_nih_billed': {
utc(2010, 6, 6, 4): 2.5
},
'label': u'mr_exciting',
'nih_used': 2.5,
'nih_bbnu': 0.0,
'nih_billed': 2.5,
'owner': u'woo',
'start': utc(2010, 6, 6, 4, 0),
'step_num': None,
},
{
# mr_exciting, step 1 (cancelled)
'date_to_nih_used': {},
'date_to_nih_bbnu': {
date(2010, 6, 6): 17.5
},
'date_to_nih_billed': {
date(2010, 6, 6): 17.5
},
'end': utc(2010, 6, 6, 4, 15),
'end_billing': utc(2010, 6, 6, 6, 0),
'hour_to_nih_used': {},
'hour_to_nih_bbnu': {
utc(2010, 6, 6, 4): 7.5,
utc(2010, 6, 6, 5): 10.0
},
'hour_to_nih_billed': {
utc(2010, 6, 6, 4): 7.5,
utc(2010, 6, 6, 5): 10.0
},
'label': u'mr_exciting',
'nih_used': 0.0,
'nih_bbnu': 17.5,
'nih_billed': 17.5,
'owner': u'woo',
'start': utc(2010, 6, 6, 4, 15),
'step_num': 1,
}
],
})
def test_pooled_cluster(self):
# same as test case above with different job keys
cluster = MockEmrObject(
bootstrapactions=[
MockEmrObject(args=[]),
MockEmrObject(args=[
MockEmrObject(
value='pool-0123456789abcdef0123456789abcdef'),
MockEmrObject(value='reflecting'),
]),
],
id='j-ISFORJOB',
name='mr_exciting.woo.20100605.232850.000000',
normalizedinstancehours='20',
state='TERMINATED',
status=MockEmrObject(
state='TERMINATED',
timeline=MockEmrObject(
creationdatetime='2010-06-05T23:30:00Z',
enddatetime='2010-06-06T01:15:00Z', # 2 hours are billed
readydatetime='2010-06-05T23:45:00Z',
),
),
steps=[
MockEmrObject(
name='mr_exciting.woo.20100605.232950.000000: Step 1 of 1',
status=MockEmrObject(
timeline=MockEmrObject(
startdatetime='2010-06-05T23:45:00Z',
enddatetime='2010-06-06T00:15:00Z',
),
),
),
MockEmrObject(
name='mr_whatever.meh.20100606.002000.000000: Step 1 of 2',
status=MockEmrObject(
timeline=MockEmrObject(
startdatetime='2010-06-06T00:30:00Z',
enddatetime='2010-06-06T00:45:00Z',
),
),
),
MockEmrObject(
name='mr_whatever.meh.20100606.002000.000000: Step 2 of 2',
status=MockEmrObject(
timeline=MockEmrObject(
startdatetime='2010-06-06T00:45:00Z',
enddatetime='2010-06-06T01:00:00Z',
),
),
),
],
)
summary = _cluster_to_full_summary(cluster)
self.assertEqual(summary, {
'created': datetime(2010, 6, 5, 23, 30),
'end': datetime(2010, 6, 6, 1, 15),
'id': u'j-ISFORJOB',
'label': u'mr_exciting',
'name': u'mr_exciting.woo.20100605.232850.000000',
'nih': 20.0,
'nih_bbnu': 7.5,
'nih_billed': 20.0,
'nih_used': 12.5,
'num_steps': 3,
'owner': u'woo',
'pool': u'reflecting',
'ran': timedelta(hours=1, minutes=45),
'ready': datetime(2010, 6, 5, 23, 45),
'state': 'TERMINATED',
'usage': [{
# bootstrapping
'date_to_nih_used': {date(2010, 6, 5): 2.5},
'date_to_nih_bbnu': {},
'date_to_nih_billed': {date(2010, 6, 5): 2.5},
'end': datetime(2010, 6, 5, 23, 45),
'end_billing': datetime(2010, 6, 5, 23, 45),
'hour_to_nih_bbnu': {},
'hour_to_nih_billed': {datetime(2010, 6, 5, 23): 2.5},
'hour_to_nih_used': {datetime(2010, 6, 5, 23): 2.5},
'label': u'mr_exciting',
'nih_used': 2.5,
'nih_bbnu': 0.0,
'nih_billed': 2.5,
'owner': u'woo',
'start': datetime(2010, 6, 5, 23, 30),
'step_num': None,
}, {
# mr_exciting, step 1 (and idle time after)
'date_to_nih_used': {date(2010, 6, 5): 2.5,
date(2010, 6, 6): 2.5},
'date_to_nih_bbnu': {date(2010, 6, 6): 2.5},
'date_to_nih_billed': {date(2010, 6, 5): 2.5,
date(2010, 6, 6): 5.0},
'end': datetime(2010, 6, 6, 0, 15),
'end_billing': datetime(2010, 6, 6, 0, 30),
'hour_to_nih_bbnu': {datetime(2010, 6, 6, 0): 2.5},
'hour_to_nih_billed': {datetime(2010, 6, 5, 23): 2.5,
datetime(2010, 6, 6, 0): 5.0},
'hour_to_nih_used': {datetime(2010, 6, 5, 23): 2.5,
datetime(2010, 6, 6, 0): 2.5},
'label': u'mr_exciting',
'nih_used': 5.0,
'nih_bbnu': 2.5,
'nih_billed': 7.5,
'owner': u'woo',
'start': datetime(2010, 6, 5, 23, 45),
'step_num': 1,
}, {
# mr whatever, step 1
'date_to_nih_used': {date(2010, 6, 6): 2.5},
'date_to_nih_bbnu': {},
'date_to_nih_billed': {date(2010, 6, 6): 2.5},
'end': datetime(2010, 6, 6, 0, 45),
'end_billing': datetime(2010, 6, 6, 0, 45),
'hour_to_nih_bbnu': {},
'hour_to_nih_billed': {datetime(2010, 6, 6, 0): 2.5},
'hour_to_nih_used': {datetime(2010, 6, 6, 0): 2.5},
'label': u'mr_whatever',
'nih_used': 2.5,
'nih_bbnu': 0.0,
'nih_billed': 2.5,
'owner': u'meh',
'start': datetime(2010, 6, 6, 0, 30),
'step_num': 1,
}, {
# mr whatever, step 2 (and idle time after)
'date_to_nih_used': {date(2010, 6, 6): 2.5},
'date_to_nih_bbnu': {date(2010, 6, 6): 5.0},
'date_to_nih_billed': {date(2010, 6, 6): 7.5},
'end': datetime(2010, 6, 6, 1, 0),
'end_billing': datetime(2010, 6, 6, 1, 30),
'hour_to_nih_bbnu': {datetime(2010, 6, 6, 1): 5.0},
'hour_to_nih_billed': {datetime(2010, 6, 6, 0): 2.5,
datetime(2010, 6, 6, 1): 5.0},
'hour_to_nih_used': {datetime(2010, 6, 6, 0): 2.5},
'label': u'mr_whatever',
'nih_used': 2.5,
'nih_bbnu': 5.0,
'nih_billed': 7.5,
'owner': u'meh',
'start': datetime(2010, 6, 6, 0, 45),
'step_num': 2,
}],
})
def test_multi_step_cluster(self):
cluster = MockEmrObject(
id='j-ISFORJOB',
name='mr_exciting.woo.20100605.232850.000000',
normalizedinstancehours='20',
status=MockEmrObject(
state='TERMINATED',
timeline=MockEmrObject(
creationdatetime='2010-06-05T23:30:00Z',
enddatetime='2010-06-06T01:15:00Z', # 2 hours are billed
readydatetime='2010-06-05T23:45:00Z',
),
),
steps=[
MockEmrObject(
name='mr_exciting.woo.20100605.232850.000000: Step 1 of 3',
status=MockEmrObject(
timeline=MockEmrObject(
startdatetime='2010-06-05T23:45:00Z',
enddatetime='2010-06-06T00:15:00Z',
),
),
),
MockEmrObject(
name='mr_exciting.woo.20100605.232850.000000: Step 2 of 3',
status=MockEmrObject(
timeline=MockEmrObject(
startdatetime='2010-06-06T00:30:00Z',
enddatetime='2010-06-06T00:45:00Z',
),
),
),
MockEmrObject(
name='mr_exciting.woo.20100605.232850.000000: Step 3 of 3',
status=MockEmrObject(
timeline=MockEmrObject(
startdatetime='2010-06-06T00:45:00Z',
enddatetime='2010-06-06T01:00:00Z',
),
),
),
],
)
summary = _cluster_to_full_summary(cluster)
self.assertEqual(summary, {
'created': datetime(2010, 6, 5, 23, 30),
'end': datetime(2010, 6, 6, 1, 15),
'id': u'j-ISFORJOB',
'label': u'mr_exciting',
'name': u'mr_exciting.woo.20100605.232850.000000',
'nih': 20.0,
'nih_bbnu': 7.5,
'nih_billed': 20.0,
'nih_used': 12.5,
'num_steps': 3,
'owner': u'woo',
'pool': None,
'ran': timedelta(hours=1, minutes=45),
'ready': datetime(2010, 6, 5, 23, 45),
'state': u'TERMINATED',
'usage': [{
# bootstrapping
'date_to_nih_used': {date(2010, 6, 5): 2.5},
'date_to_nih_bbnu': {},
'date_to_nih_billed': {date(2010, 6, 5): 2.5},
'end': datetime(2010, 6, 5, 23, 45),
'end_billing': datetime(2010, 6, 5, 23, 45),
'hour_to_nih_bbnu': {},
'hour_to_nih_billed': {datetime(2010, 6, 5, 23): 2.5},
'hour_to_nih_used': {datetime(2010, 6, 5, 23): 2.5},
'label': u'mr_exciting',
'nih_used': 2.5,
'nih_bbnu': 0.0,
'nih_billed': 2.5,
'owner': u'woo',
'start': datetime(2010, 6, 5, 23, 30),
'step_num': None,
}, {
# step 1 (and idle time after)
'date_to_nih_used': {date(2010, 6, 5): 2.5,
date(2010, 6, 6): 2.5},
'date_to_nih_bbnu': {date(2010, 6, 6): 2.5},
'date_to_nih_billed': {date(2010, 6, 5): 2.5,
date(2010, 6, 6): 5.0},
'end': datetime(2010, 6, 6, 0, 15),
'end_billing': datetime(2010, 6, 6, 0, 30),
'hour_to_nih_bbnu': {datetime(2010, 6, 6, 0): 2.5},
'hour_to_nih_billed': {datetime(2010, 6, 5, 23): 2.5,
datetime(2010, 6, 6, 0): 5.0},
'hour_to_nih_used': {datetime(2010, 6, 5, 23): 2.5,
datetime(2010, 6, 6, 0): 2.5},
'label': u'mr_exciting',
'nih_used': 5.0,
'nih_bbnu': 2.5,
'nih_billed': 7.5,
'owner': u'woo',
'start': datetime(2010, 6, 5, 23, 45),
'step_num': 1,
}, {
# step 2
'date_to_nih_used': {date(2010, 6, 6): 2.5},
'date_to_nih_bbnu': {},
'date_to_nih_billed': {date(2010, 6, 6): 2.5},
'end': datetime(2010, 6, 6, 0, 45),
'end_billing': datetime(2010, 6, 6, 0, 45),
'hour_to_nih_bbnu': {},
'hour_to_nih_billed': {datetime(2010, 6, 6, 0): 2.5},
'hour_to_nih_used': {datetime(2010, 6, 6, 0): 2.5},
'label': u'mr_exciting',
'nih_used': 2.5,
'nih_bbnu': 0.0,
'nih_billed': 2.5,
'owner': u'woo',
'start': datetime(2010, 6, 6, 0, 30),
'step_num': 2,
}, {
# step 3 (and idle time after)
'date_to_nih_used': {date(2010, 6, 6): 2.5},
'date_to_nih_bbnu': {date(2010, 6, 6): 5.0},
'date_to_nih_billed': {date(2010, 6, 6): 7.5},
'end': datetime(2010, 6, 6, 1, 0),
'end_billing': datetime(2010, 6, 6, 1, 30),
'hour_to_nih_bbnu': {datetime(2010, 6, 6, 1): 5.0},
'hour_to_nih_billed': {datetime(2010, 6, 6, 0): 2.5,
datetime(2010, 6, 6, 1): 5.0},
'hour_to_nih_used': {datetime(2010, 6, 6, 0): 2.5},
'label': u'mr_exciting',
'nih_used': 2.5,
'nih_bbnu': 5.0,
'nih_billed': 7.5,
'owner': u'woo',
'start': datetime(2010, 6, 6, 0, 45),
'step_num': 3,
}],
})
def test_cluster_with_one_cancelled_step(self):
cluster = MockEmrObject(
id='j-ISFORJACUZZI',
name='mr_exciting.woo.20100606.035855.000000',
normalizedinstancehours='20',
status=MockEmrObject(
state='RUNNING',
timeline=MockEmrObject(
creationdatetime='2010-06-06T04:00:00Z',
enddatetime='2010-06-06T05:30:00Z',
readydatetime='2010-06-06T04:15:00Z',
),
),
# step doesn't have end time even though cluster does
steps=[
MockEmrObject(
name='mr_exciting.woo.20100606.035855.000000: Step 1 of 3',
status=MockEmrObject(
timeline=MockEmrObject(
startdatetime='2010-06-06T04:15:00Z',
),
),
),
]
)
summary = _cluster_to_full_summary(cluster)
self.assertEqual(summary, {
'created': datetime(2010, 6, 6, 4, 0),
'end': datetime(2010, 6, 6, 5, 30),
'id': u'j-ISFORJACUZZI',
'label': u'mr_exciting',
'name': u'mr_exciting.woo.20100606.035855.000000',
'nih': 20.0,
'nih_bbnu': 17.5,
'nih_billed': 20.0,
'nih_used': 2.5,
'num_steps': 1,
'owner': u'woo',
'pool': None,
'ran': timedelta(hours=1, minutes=30),
'ready': datetime(2010, 6, 6, 4, 15),
'state': u'RUNNING',
'usage': [{
# bootstrapping
'date_to_nih_used': {date(2010, 6, 6): 2.5},
'date_to_nih_bbnu': {},
'date_to_nih_billed': {date(2010, 6, 6): 2.5},
'end': datetime(2010, 6, 6, 4, 15),
'end_billing': datetime(2010, 6, 6, 4, 15),
'hour_to_nih_used': {datetime(2010, 6, 6, 4): 2.5},
'hour_to_nih_bbnu': {},
'hour_to_nih_billed': {datetime(2010, 6, 6, 4): 2.5},
'label': u'mr_exciting',
'nih_used': 2.5,
'nih_bbnu': 0.0,
'nih_billed': 2.5,
'owner': u'woo',
'start': datetime(2010, 6, 6, 4, 0),
'step_num': None,
}, {
# mr_exciting, step 1 (cancelled)
'date_to_nih_used': {},
'date_to_nih_bbnu': {date(2010, 6, 6): 17.5},
'date_to_nih_billed': {date(2010, 6, 6): 17.5},
'end': datetime(2010, 6, 6, 4, 15),
'end_billing': datetime(2010, 6, 6, 6, 0),
'hour_to_nih_used': {},
'hour_to_nih_bbnu': {datetime(2010, 6, 6, 4): 7.5,
datetime(2010, 6, 6, 5): 10.0},
'hour_to_nih_billed': {datetime(2010, 6, 6, 4): 7.5,
datetime(2010, 6, 6, 5): 10.0},
'label': u'mr_exciting',
'nih_used': 0.0,
'nih_bbnu': 17.5,
'nih_billed': 17.5,
'owner': u'woo',
'start': datetime(2010, 6, 6, 4, 15),
'step_num': 1,
}],
})
def test_cluster_with_one_cancelled_step(self):
cluster = dict(
BootstrapActions=[],
Id='j-ISFORJACUZZI',
Name='mr_exciting.woo.20100606.035855.000000',
NormalizedInstanceHours=20,
Status=dict(
State='RUNNING',
Timeline=dict(
CreationDateTime=parse('2010-06-06T04:00:00Z'),
EndDateTime=parse('2010-06-06T05:30:00Z'),
ReadyDateTime=parse('2010-06-06T04:15:00Z'),
),
),
# step doesn't have end time even though cluster does
Steps=[
dict(
Name='mr_exciting.woo.20100606.035855.000000: Step 1 of 3',
Status=dict(
Timeline=dict(
StartDateTime=parse('2010-06-06T04:15:00Z'),
),
),
),
]
)
summary = _cluster_to_full_summary(cluster)
self.assertEqual(summary, {
'created': utc(2010, 6, 6, 4, 0),
'end': utc(2010, 6, 6, 5, 30),
'id': u'j-ISFORJACUZZI',
'label': u'mr_exciting',
'name': u'mr_exciting.woo.20100606.035855.000000',
'nih': 20.0,
'nih_bbnu': 12.5,
'nih_billed': 15.0,
'nih_used': 2.5,
'num_steps': 1,
'owner': u'woo',
'pool': None,
'ran': timedelta(hours=1, minutes=30),
'ready': utc(2010, 6, 6, 4, 15),
'state': u'RUNNING',
'usage': [{
# bootstrapping
'date_to_nih_used': {date(2010, 6, 6): 2.5},
'date_to_nih_bbnu': {},
'date_to_nih_billed': {date(2010, 6, 6): 2.5},
'end': utc(2010, 6, 6, 4, 15),
'end_billing': utc(2010, 6, 6, 4, 15),
'hour_to_nih_used': {utc(2010, 6, 6, 4): 2.5},
'hour_to_nih_bbnu': {},
'hour_to_nih_billed': {utc(2010, 6, 6, 4): 2.5},
'label': u'mr_exciting',
'nih_used': 2.5,
'nih_bbnu': 0.0,
'nih_billed': 2.5,
'owner': u'woo',
'start': utc(2010, 6, 6, 4, 0),
'step_num': None,
}, {
# mr_exciting, step 1 (cancelled)
'date_to_nih_used': {},
'date_to_nih_bbnu': {date(2010, 6, 6): 12.5},
'date_to_nih_billed': {date(2010, 6, 6): 12.5},
'end': utc(2010, 6, 6, 4, 15),
'end_billing': utc(2010, 6, 6, 5, 30),
'hour_to_nih_used': {},
'hour_to_nih_bbnu': {utc(2010, 6, 6, 4): 7.5,
utc(2010, 6, 6, 5): 5.0},
'hour_to_nih_billed': {utc(2010, 6, 6, 4): 7.5,
utc(2010, 6, 6, 5): 5.0},
'label': u'mr_exciting',
'nih_used': 0.0,
'nih_bbnu': 12.5,
'nih_billed': 12.5,
'owner': u'woo',
'start': utc(2010, 6, 6, 4, 15),
'step_num': 1,
}],
})
def test_pooled_cluster(self):
# same as test case above with different job keys
cluster = dict(
BootstrapActions=[
dict(Args=[], Name='empty'),
dict(Args=[
'pool-0123456789abcdef0123456789abcdef', 'reflecting'
],
Name='master'),
],
Id='j-ISFORJOB',
Name='mr_exciting.woo.20100605.232850.000000',
NormalizedInstanceHours=20,
State='TERMINATED',
Status=dict(
State='TERMINATED',
Timeline=dict(
CreationDateTime=parse('2010-06-05T23:30:00Z'),
EndDateTime=parse('2010-06-06T01:15:00Z'), # 2 hrs billed
ReadyDateTime=parse('2010-06-05T23:45:00Z'),
),
),
Steps=[
dict(
Name='mr_exciting.woo.20100605.232950.000000: Step 1 of 1',
Status=dict(Timeline=dict(
StartDateTime=parse('2010-06-05T23:45:00Z'),
EndDateTime=parse('2010-06-06T00:15:00Z'),
), ),
),
dict(
Name='mr_whatever.meh.20100606.002000.000000: Step 1 of 2',
Status=dict(Timeline=dict(
StartDateTime=parse('2010-06-06T00:30:00Z'),
EndDateTime=parse('2010-06-06T00:45:00Z'),
), ),
),
dict(
Name='mr_whatever.meh.20100606.002000.000000: Step 2 of 2',
Status=dict(Timeline=dict(
StartDateTime=parse('2010-06-06T00:45:00Z'),
EndDateTime=parse('2010-06-06T01:00:00Z'),
), ),
),
],
)
summary = _cluster_to_full_summary(cluster)
self.assertEqual(
summary,
{
'created':
utc(2010, 6, 5, 23, 30),
'end':
utc(2010, 6, 6, 1, 15),
'id':
u'j-ISFORJOB',
'label':
u'mr_exciting',
'name':
u'mr_exciting.woo.20100605.232850.000000',
'nih':
20.0,
'nih_bbnu':
7.5,
'nih_billed':
20.0,
'nih_used':
12.5,
'num_steps':
3,
'owner':
u'woo',
'pool':
u'reflecting',
'ran':
timedelta(hours=1, minutes=45),
'ready':
utc(2010, 6, 5, 23, 45),
'state':
'TERMINATED',
'usage': [
{
# bootstrapping
'date_to_nih_used': {
date(2010, 6, 5): 2.5
},
'date_to_nih_bbnu': {},
'date_to_nih_billed': {
date(2010, 6, 5): 2.5
},
'end': utc(2010, 6, 5, 23, 45),
'end_billing': utc(2010, 6, 5, 23, 45),
'hour_to_nih_bbnu': {},
'hour_to_nih_billed': {
utc(2010, 6, 5, 23): 2.5
},
'hour_to_nih_used': {
utc(2010, 6, 5, 23): 2.5
},
'label': u'mr_exciting',
'nih_used': 2.5,
'nih_bbnu': 0.0,
'nih_billed': 2.5,
'owner': u'woo',
'start': utc(2010, 6, 5, 23, 30),
'step_num': None,
},
{
# mr_exciting, step 1 (and idle time after)
'date_to_nih_used': {
date(2010, 6, 5): 2.5,
date(2010, 6, 6): 2.5
},
'date_to_nih_bbnu': {
date(2010, 6, 6): 2.5
},
'date_to_nih_billed': {
date(2010, 6, 5): 2.5,
date(2010, 6, 6): 5.0
},
'end': utc(2010, 6, 6, 0, 15),
'end_billing': utc(2010, 6, 6, 0, 30),
'hour_to_nih_bbnu': {
utc(2010, 6, 6, 0): 2.5
},
'hour_to_nih_billed': {
utc(2010, 6, 5, 23): 2.5,
utc(2010, 6, 6, 0): 5.0
},
'hour_to_nih_used': {
utc(2010, 6, 5, 23): 2.5,
utc(2010, 6, 6, 0): 2.5
},
'label': u'mr_exciting',
'nih_used': 5.0,
'nih_bbnu': 2.5,
'nih_billed': 7.5,
'owner': u'woo',
'start': utc(2010, 6, 5, 23, 45),
'step_num': 1,
},
{
# mr whatever, step 1
'date_to_nih_used': {
date(2010, 6, 6): 2.5
},
'date_to_nih_bbnu': {},
'date_to_nih_billed': {
date(2010, 6, 6): 2.5
},
'end': utc(2010, 6, 6, 0, 45),
'end_billing': utc(2010, 6, 6, 0, 45),
'hour_to_nih_bbnu': {},
'hour_to_nih_billed': {
utc(2010, 6, 6, 0): 2.5
},
'hour_to_nih_used': {
utc(2010, 6, 6, 0): 2.5
},
'label': u'mr_whatever',
'nih_used': 2.5,
'nih_bbnu': 0.0,
'nih_billed': 2.5,
'owner': u'meh',
'start': utc(2010, 6, 6, 0, 30),
'step_num': 1,
},
{
# mr whatever, step 2 (and idle time after)
'date_to_nih_used': {
date(2010, 6, 6): 2.5
},
'date_to_nih_bbnu': {
date(2010, 6, 6): 5.0
},
'date_to_nih_billed': {
date(2010, 6, 6): 7.5
},
'end': utc(2010, 6, 6, 1, 0),
'end_billing': utc(2010, 6, 6, 1, 30),
'hour_to_nih_bbnu': {
utc(2010, 6, 6, 1): 5.0
},
'hour_to_nih_billed': {
utc(2010, 6, 6, 0): 2.5,
utc(2010, 6, 6, 1): 5.0
},
'hour_to_nih_used': {
utc(2010, 6, 6, 0): 2.5
},
'label': u'mr_whatever',
'nih_used': 2.5,
'nih_bbnu': 5.0,
'nih_billed': 7.5,
'owner': u'meh',
'start': utc(2010, 6, 6, 0, 45),
'step_num': 2,
}
],
})
def _test_new_or_legacy_pooled_cluster(self, **kwargs):
# same as test case above with different job keys
cluster = dict(
Id='j-ISFORJOB',
Name='mr_exciting.woo.20100605.232850.000000',
NormalizedInstanceHours=20,
State='TERMINATED',
Status=dict(
State='TERMINATED',
Timeline=dict(
CreationDateTime=parse('2010-06-05T23:30:00Z'),
EndDateTime=parse('2010-06-06T01:15:00Z'),
ReadyDateTime=parse('2010-06-05T23:45:00Z'),
),
),
Steps=[
dict(
Name='mr_exciting.woo.20100605.232950.000000: Step 1 of 1',
Status=dict(
Timeline=dict(
StartDateTime=parse('2010-06-05T23:45:00Z'),
EndDateTime=parse('2010-06-06T00:15:00Z'),
),
),
),
dict(
Name='mr_whatever.meh.20100606.002000.000000: Step 1 of 2',
Status=dict(
Timeline=dict(
StartDateTime=parse('2010-06-06T00:30:00Z'),
EndDateTime=parse('2010-06-06T00:45:00Z'),
),
),
),
dict(
Name='mr_whatever.meh.20100606.002000.000000: Step 2 of 2',
Status=dict(
Timeline=dict(
StartDateTime=parse('2010-06-06T00:45:00Z'),
EndDateTime=parse('2010-06-06T01:00:00Z'),
),
),
),
],
**kwargs
)
summary = _cluster_to_full_summary(cluster)
self.assertEqual(summary, {
'created': utc(2010, 6, 5, 23, 30),
'end': utc(2010, 6, 6, 1, 15),
'id': u'j-ISFORJOB',
'label': u'mr_exciting',
'name': u'mr_exciting.woo.20100605.232850.000000',
'nih': 20.0,
'nih_bbnu': 5.0,
'nih_billed': 17.5,
'nih_used': 12.5,
'num_steps': 3,
'owner': u'woo',
'pool': u'reflecting',
'ran': timedelta(hours=1, minutes=45),
'ready': utc(2010, 6, 5, 23, 45),
'state': 'TERMINATED',
'usage': [{
# bootstrapping
'date_to_nih_used': {date(2010, 6, 5): 2.5},
'date_to_nih_bbnu': {},
'date_to_nih_billed': {date(2010, 6, 5): 2.5},
'end': utc(2010, 6, 5, 23, 45),
'end_billing': utc(2010, 6, 5, 23, 45),
'hour_to_nih_bbnu': {},
'hour_to_nih_billed': {utc(2010, 6, 5, 23): 2.5},
'hour_to_nih_used': {utc(2010, 6, 5, 23): 2.5},
'label': u'mr_exciting',
'nih_used': 2.5,
'nih_bbnu': 0.0,
'nih_billed': 2.5,
'owner': u'woo',
'start': utc(2010, 6, 5, 23, 30),
'step_num': None,
}, {
# mr_exciting, step 1 (and idle time after)
'date_to_nih_used': {date(2010, 6, 5): 2.5,
date(2010, 6, 6): 2.5},
'date_to_nih_bbnu': {date(2010, 6, 6): 2.5},
'date_to_nih_billed': {date(2010, 6, 5): 2.5,
date(2010, 6, 6): 5.0},
'end': utc(2010, 6, 6, 0, 15),
'end_billing': utc(2010, 6, 6, 0, 30),
'hour_to_nih_bbnu': {utc(2010, 6, 6, 0): 2.5},
'hour_to_nih_billed': {utc(2010, 6, 5, 23): 2.5,
utc(2010, 6, 6, 0): 5.0},
'hour_to_nih_used': {utc(2010, 6, 5, 23): 2.5,
utc(2010, 6, 6, 0): 2.5},
'label': u'mr_exciting',
'nih_used': 5.0,
'nih_bbnu': 2.5,
'nih_billed': 7.5,
'owner': u'woo',
'start': utc(2010, 6, 5, 23, 45),
'step_num': 1,
}, {
# mr whatever, step 1
'date_to_nih_used': {date(2010, 6, 6): 2.5},
'date_to_nih_bbnu': {},
'date_to_nih_billed': {date(2010, 6, 6): 2.5},
'end': utc(2010, 6, 6, 0, 45),
'end_billing': utc(2010, 6, 6, 0, 45),
'hour_to_nih_bbnu': {},
'hour_to_nih_billed': {utc(2010, 6, 6, 0): 2.5},
'hour_to_nih_used': {utc(2010, 6, 6, 0): 2.5},
'label': u'mr_whatever',
'nih_used': 2.5,
'nih_bbnu': 0.0,
'nih_billed': 2.5,
'owner': u'meh',
'start': utc(2010, 6, 6, 0, 30),
'step_num': 1,
}, {
# mr whatever, step 2 (and idle time after)
'date_to_nih_used': {date(2010, 6, 6): 2.5},
'date_to_nih_bbnu': {date(2010, 6, 6): 2.5},
'date_to_nih_billed': {date(2010, 6, 6): 5.0},
'end': utc(2010, 6, 6, 1, 0),
'end_billing': utc(2010, 6, 6, 1, 15),
'hour_to_nih_bbnu': {utc(2010, 6, 6, 1): 2.5},
'hour_to_nih_billed': {utc(2010, 6, 6, 0): 2.5,
utc(2010, 6, 6, 1): 2.5},
'hour_to_nih_used': {utc(2010, 6, 6, 0): 2.5},
'label': u'mr_whatever',
'nih_used': 2.5,
'nih_bbnu': 2.5,
'nih_billed': 5.0,
'owner': u'meh',
'start': utc(2010, 6, 6, 0, 45),
'step_num': 2,
}],
})
def test_still_bootstrapping_cluster_with_no_steps(self):
cluster = MockEmrObject(
id='j-ISFORJOKE',
name='mr_exciting.woo.20100605.235850.000000',
normalizedinstancehours='10',
status=MockEmrObject(
state='BOOTSTRAPPING',
timeline=MockEmrObject(
creationdatetime='2010-06-06T00:00:00Z', ),
),
)
summary = _cluster_to_full_summary(cluster,
now=datetime(2010, 6, 6, 0, 30))
self.assertEqual(
summary, {
'created':
datetime(2010, 6, 6, 0, 0),
'end':
None,
'id':
u'j-ISFORJOKE',
'label':
u'mr_exciting',
'name':
u'mr_exciting.woo.20100605.235850.000000',
'nih':
10.0,
'nih_bbnu':
0.0,
'nih_billed':
5.0,
'nih_used':
5.0,
'num_steps':
0,
'owner':
u'woo',
'pool':
None,
'ran':
timedelta(minutes=30),
'ready':
None,
'state':
u'BOOTSTRAPPING',
'usage': [{
'date_to_nih_bbnu': {},
'date_to_nih_billed': {
date(2010, 6, 6): 5.0
},
'date_to_nih_used': {
date(2010, 6, 6): 5.0
},
'end': datetime(2010, 6, 6, 0, 30),
'end_billing': datetime(2010, 6, 6, 0, 30),
'hour_to_nih_bbnu': {},
'hour_to_nih_billed': {
datetime(2010, 6, 6, 0): 5.0
},
'hour_to_nih_used': {
datetime(2010, 6, 6, 0): 5.0
},
'label': u'mr_exciting',
'nih_bbnu': 0.0,
'nih_billed': 5.0,
'nih_used': 5.0,
'owner': u'woo',
'start': datetime(2010, 6, 6, 0, 0),
'step_num': None,
}],
})
