def _run_step_in_regions(self, action, step, regions):
"""
Called from :py:meth:`~.run`; run a given step in all applicable /
specified regions.
:param action: Name of the action to do, "run" or "dryrun"
:type action: str
:param step: A reference to the :py:class:`~.BaseStep` subclass to run
:type step: object
:param regions: list of string region names to run in
:type regions: list
"""
for r_idx, region_name in enumerate(regions):
if step.name in ['policygen', 'dryrun-diff']:
# Some steps need a config with %%AWS_REGION%% un-interpolated
region_conf = self.config
else:
region_conf = self.config.config_for_region(region_name)
if not step.run_in_region(region_name, region_conf):
logger.info(
bold('SKIPPING Step %s in REGION %d of %d (%s)' %
(step.name, r_idx + 1, len(regions), region_name)))
continue
logger.info(
bold('Step %s in REGION %d of %d (%s)' %
(step.name, r_idx + 1, len(regions), region_name)))
if action == 'run':
step(region_name, region_conf).run()
else:
step(region_name, region_conf).dryrun()
sys.stdout.flush()
sys.stderr.flush()
def test_run_in_regions_dryrun_skip_some(self):
m_conf = Mock(spec_set=ManheimConfig)
m_conf_r1 = Mock(spec_set=ManheimConfig)
m_conf_r2 = Mock(spec_set=ManheimConfig)
m_conf_r3 = Mock(spec_set=ManheimConfig)
def se_conf_for_region(rname):
if rname == 'r1':
return m_conf_r1
if rname == 'r2':
return m_conf_r2
if rname == 'r3':
return m_conf_r3
m_conf.config_for_region.side_effect = se_conf_for_region
with patch('%s.logger' % pbm, autospec=True) as mock_logger:
with patch('%s.ManheimConfig.from_file' % pbm) as mock_cff:
mock_cff.return_value = m_conf
runner.CustodianRunner('acctName')._run_step_in_regions(
'dryrun', self.cls2, ['r2', 'r3'])
assert self.cls2.mock_calls == [
call.run_in_region('r2', m_conf_r2),
call.run_in_region('r3', m_conf_r3),
call('r3', m_conf_r3),
call().dryrun()
]
assert m_conf.config_for_region.mock_calls == [call('r2'), call('r3')]
assert mock_logger.mock_calls == [
call.info(bold('SKIPPING Step cls2 in REGION 1 of 2 (r2)')),
call.info(bold('Step cls2 in REGION 2 of 2 (r3)'))
]
def run(self, action, regions=[], step_names=[], skip_steps=[]):
"""
Main method to run all steps. This calls :py:meth:`~._steps_to_run`
to determine which step classes to run and the order to run them in,
and then loops through that list calling the :py:meth:`~.BaseStep.run`
or :py:meth:`~.BaseStep.dryrun` method on each of them, according to the
``action`` specified.
:param action: Name of the action to do, "run" or "dryrun"
:type action: str
:param regions: list of string region names to run in; if left empty,
run in all regions listed in config file
:type regions: list
:param step_names: list of string step names to run; if not specified,
will run all defined steps. Steps are always run in the order defined
in :py:attr:`~.ordered_step_classes`.
:type step_names: list
:param skip_steps: list of string step names to skip running
:type skip_steps: list
"""
self._validate_account()
to_run = self._steps_to_run(step_names, skip_steps)
if to_run == self.ordered_step_classes:
logger.info(bold(
'Beginning %s - %d steps' % (action, len(to_run))
))
else:
logger.info(bold(
'Beginning %s - %d of %d steps selected' % (
action, len(to_run), len(self.ordered_step_classes)
)
))
if regions:
if not set(regions).issubset(set(self.config.regions)):
raise RuntimeError(
'ERROR: All specified region names must be listed in the '
'"regions" section of the config file '
'(%s)' % self._config_path
)
else:
# use all regions from config file
regions = self.config.regions
for idx, step in enumerate(to_run):
logger.info(bold(
'Step %d of %d - %s' % (idx + 1, len(to_run), step.name)
))
self._run_step_in_regions(action, step, regions)
logger.info(bold('SUCCESS: All %d steps complete!' % len(to_run)))
def test_run_invalid_region_name(self):
m_conf = Mock(spec_set=ManheimConfig)
type(m_conf).regions = PropertyMock(return_value=['r1', 'r2', 'r3'])
with patch('%s.CustodianRunner.ordered_step_classes' % pbm,
self.steps):
with patch.multiple('%s.CustodianRunner' % pbm,
autospec=True,
_steps_to_run=DEFAULT,
_run_step_in_regions=DEFAULT,
_validate_account=DEFAULT) as mocks:
mocks['_steps_to_run'].return_value = [self.cls2, self.cls3]
with patch('%s.logger' % pbm, autospec=True) as mock_logger:
with patch('%s.ManheimConfig.from_file' % pbm) as mock_cff:
mock_cff.return_value = m_conf
with pytest.raises(RuntimeError) as exc:
cls = runner.CustodianRunner('acctName')
cls.run('dryrun',
regions=['notValid'],
step_names=['cls2', 'cls3', 'cls4'],
skip_steps=['cls4'])
assert str(exc.value) == 'ERROR: All specified region names must be ' \
'listed in the "regions" section of the ' \
'config file (manheim-c7n-tools.yml)'
assert mocks['_steps_to_run'].mock_calls == [
call(cls, ['cls2', 'cls3', 'cls4'], ['cls4'])
]
assert mocks['_run_step_in_regions'].mock_calls == []
assert self.cls1.mock_calls == []
assert self.cls2.mock_calls == []
assert self.cls3.mock_calls == []
assert self.cls4.mock_calls == []
assert mock_logger.mock_calls == [
call.info(bold('Beginning dryrun - 2 of 4 steps selected'))
]
assert mocks['_validate_account'].mock_calls == [call(cls)]
def test_run_in_regions_policygen_run(self):
m_conf = Mock(spec_set=ManheimConfig)
m_conf_r1 = Mock(spec_set=ManheimConfig)
m_conf_r2 = Mock(spec_set=ManheimConfig)
m_conf_r3 = Mock(spec_set=ManheimConfig)
def se_conf_for_region(rname):
if rname == 'r1':
return m_conf_r1
if rname == 'r2':
return m_conf_r2
if rname == 'r3':
return m_conf_r3
m_conf.config_for_region.side_effect = se_conf_for_region
with patch('%s.logger' % pbm, autospec=True) as mock_logger:
with patch('%s.ManheimConfig.from_file' % pbm) as mock_cff:
mock_cff.return_value = m_conf
with patch('%s.PolicygenStep' % pbm,
autospec=True) as mock_pgs:
type(mock_pgs).name = PropertyMock(
return_value='policygen')
mock_pgs.run_in_region.return_value = True
runner.CustodianRunner('acctName')._run_step_in_regions(
'run', mock_pgs, ['r1', 'r2', 'r3'])
assert mock_pgs.mock_calls == [
call.run_in_region('r1', m_conf),
call('r1', m_conf),
call().run(),
call.run_in_region('r2', m_conf),
call('r2', m_conf),
call().run(),
call.run_in_region('r3', m_conf),
call('r3', m_conf),
call().run()
]
assert m_conf.config_for_region.mock_calls == []
assert mock_logger.mock_calls == [
call.info(bold('Step policygen in REGION 1 of 3 (r1)')),
call.info(bold('Step policygen in REGION 2 of 3 (r2)')),
call.info(bold('Step policygen in REGION 3 of 3 (r3)'))
]
def test_run_dryrun_some_steps_some_regions(self):
m_conf = Mock(spec_set=ManheimConfig)
type(m_conf).regions = PropertyMock(return_value=['r1', 'r2', 'r3'])
with patch('%s.CustodianRunner.ordered_step_classes' % pbm,
self.steps):
with patch.multiple('%s.CustodianRunner' % pbm,
autospec=True,
_steps_to_run=DEFAULT,
_run_step_in_regions=DEFAULT,
_validate_account=DEFAULT) as mocks:
mocks['_steps_to_run'].return_value = [self.cls2, self.cls3]
with patch('%s.logger' % pbm, autospec=True) as mock_logger:
with patch('%s.ManheimConfig.from_file' % pbm) as mock_cff:
mock_cff.return_value = m_conf
cls = runner.CustodianRunner('aName')
cls.run('dryrun',
regions=['r2'],
step_names=['cls2', 'cls3', 'cls4'],
skip_steps=['cls4'])
assert mocks['_steps_to_run'].mock_calls == [
call(cls, ['cls2', 'cls3', 'cls4'], ['cls4'])
]
assert mocks['_run_step_in_regions'].mock_calls == [
call(cls, 'dryrun', self.cls2, ['r2']),
call(cls, 'dryrun', self.cls3, ['r2'])
]
assert self.cls1.mock_calls == []
assert self.cls2.mock_calls == []
assert self.cls3.mock_calls == []
assert self.cls4.mock_calls == []
assert mock_logger.mock_calls == [
call.info(bold('Beginning dryrun - 2 of 4 steps selected')),
call.info(bold('Step 1 of 2 - cls2')),
call.info(bold('Step 2 of 2 - cls3')),
call.info(bold('SUCCESS: All 2 steps complete!'))
]
assert mock_cff.mock_calls == [call('manheim-c7n-tools.yml', 'aName')]
assert mocks['_validate_account'].mock_calls == [call(cls)]
def test_bold(self):
assert bold('foo') == "\033[1mfoo\033[0m"
