def setUp(self):
"""
Setup a mocked GoCDAPI client for use in tests.
"""
with patch('yagocd.session.Session', spec=Session):
self.test_gocd_client = GoCDAPI('user', 'password', 'http://gocd')
super(GoCDApiTestCase, self).setUp()
def find_and_advance_pipeline(gocd_user, gocd_password, gocd_url,
hipchat_token, hipchat_channel, pipeline, stage,
relative_dt, out_file):
"""
Find the GoCD advancement pipeline that should be advanced/deployed to production - and advance it.
"""
gocd = GoCDAPI(gocd_user, gocd_password, gocd_url)
# If a datetime string was passed-in, convert it to a datetime.
if relative_dt:
relative_dt = parser.parse(relative_dt)
pipeline_to_advance = gocd.fetch_pipeline_to_advance(
pipeline, stage, relative_dt)
gocd.approve_stage(pipeline_to_advance.name, pipeline_to_advance.counter,
stage)
advance_info = {
'name': pipeline_to_advance.name,
'counter': pipeline_to_advance.counter,
'stage': stage,
'url': pipeline_to_advance.url
}
LOG.info('Successfully advanced this pipeline: %s', advance_info)
dirname = os.path.dirname(out_file.name)
if dirname:
os.makedirs(dirname, exist_ok=True)
yaml.safe_dump(advance_info, stream=out_file)
if hipchat_token:
submit_hipchat_message(
hipchat_token, hipchat_channel,
'PROD DEPLOY: Pipeline was advanced: {}'.format(
pipeline_to_advance.url), "green")
def update_recent_deployers(opsgenie_api_key,
opsgenie_team_id,
gocd_pipelines,
gocd_username,
gocd_password,
gocd_url,
github_token,
recent_cutoff=30):
"""
Update an OpsGenie team to contain only those users whose changes were recently deployed
by a particular GoCD pipeline.
"""
repo_tools_repo = GitHubAPI('edx', 'repo-tools-data', github_token)
people_yaml_data = repo_tools_repo.file_contents('people.yaml')
people_yaml = yaml.safe_load(people_yaml_data)
email_aliases = {
gh: [user['email']] + user.get('other_emails', [])
for gh, user in people_yaml.items()
}
gocd = GoCDAPI(gocd_username, gocd_password, gocd_url)
recent_deployers = set().union(*(gocd.recent_deployers(
pipeline, stages, cutoff=recent_cutoff, email_aliases=email_aliases)
for pipeline, stages in gocd_pipelines))
opsgenie = OpsGenieAPI(opsgenie_api_key)
while True:
try:
opsgenie.set_team_members(opsgenie_team_id, recent_deployers)
break
except HTTPError as exc:
if exc.response.status_code == 422:
message = exc.response.json().get('message')
if message is None:
raise
match = re.match(
r"No user exists with username \[(?P<user>.*)\]", message)
if match is None:
raise
user = match.group('user')
click.echo(
click.style('Removing user {!r} and retrying'.format(user),
fg='red'))
recent_deployers.remove(user)
continue
else:
raise
class GoCDApiTestCase(TestCase):
"""
Tests the functionality of the GoCD API.
All network calls are mocked out.
"""
_instance_map = {}
def setUp(self):
"""
Setup a mocked GoCDAPI client for use in tests.
"""
with patch('yagocd.session.Session', spec=Session):
self.test_gocd_client = GoCDAPI('user', 'password', 'http://gocd')
super(GoCDApiTestCase, self).setUp()
def _build_gets(self, gets):
"""
Build up the PipelineInstance objects to return.
"""
with patch('yagocd.session.Session', spec=Session) as mock_session:
for instance in gets:
self._instance_map[(instance['name'], instance['counter'])] = PipelineInstance(mock_session, instance)
def _mock_gets(self, name, counter):
"""
Return a mocked PipelineInstance based on pipeline instance name & counter.
"""
return self._instance_map[(name, counter)]
def _build_pipeline_system_data(self, fillin_params):
"""
Build the mocked JSON data that would be returned from the GoCD api.
Many keys/values are left out - only the essential ones for testing remain.
"""
return (
{
'name': 'prerelease_edxapp_materials_latest',
'counter': fillin_params['prerelease_counter'],
'stages': [
{
'name': 'initial_verification',
'jobs': [
{'name': 'armed_job', 'scheduled_date': fillin_params['job1_trigger_time']}
],
'scheduled': fillin_params['stage_status']
},
{
'name': 'manual_verification',
'jobs': [],
'scheduled': fillin_params['stage_status']
}
],
},
{
'name': 'manual_verification_edxapp_prod_early_ami_build',
'counter': fillin_params['manual_counter'],
'stages': [
{
'name': 'initial_verification',
'jobs': [
{'name': 'armed_job', 'scheduled_date': fillin_params['job2_trigger_time']}
],
'scheduled': fillin_params['stage_status']
},
{
'name': 'manual_verification',
'jobs': [],
'scheduled': fillin_params['stage_status']
}
],
}
)
def _build_mocked_gocd_data(self, fillin_params):
"""
Setup the GoCD data mocking for a single test.
"""
instances = []
with patch('yagocd.session.Session', spec=Session) as mock_session:
for fillins in fillin_params:
inst_data = self._build_pipeline_system_data(fillins)
# Build the value_stream_map() to return for the pipeline instance.
vsm_data = [PipelineInstance(mock_session, instance) for instance in inst_data]
# Build a mocked PipelineInstance for the manual_verification pipeline
# and add it to a full_history() list.
mock_instance = PipelineInstance(mock_session, inst_data[1])
mock_instance.value_stream_map = Mock(return_value=vsm_data)
instances.append(mock_instance)
# Add the PipelineInstance data to the mocked get() call.
self._build_gets(inst_data)
return instances
def _test_pipeline_instance_finding(
self, manual_trigger_time, stage_statuses, exception_expected=None, current_time=None
):
"""
Test pipeline instance finding using common code for the tests below.
"""
gocd_api_data = [
{
'prerelease_counter': 238,
'manual_counter': 157,
'job1_trigger_time': 1487707461420,
'job2_trigger_time': 1487707461420 + FAKE_TIME_BETWEEN_PIPELINE_RUNS_MS,
'stage_status': stage_statuses[0]
},
{
'prerelease_counter': 228,
'manual_counter': 148,
'job1_trigger_time': manual_trigger_time,
'job2_trigger_time': manual_trigger_time + FAKE_TIME_BETWEEN_PIPELINE_RUNS_MS,
'stage_status': stage_statuses[1]
}
]
instances = self._build_mocked_gocd_data(gocd_api_data)
with patch.object(self.test_gocd_client.client.pipelines, 'full_history', return_value=instances):
with patch.object(self.test_gocd_client.client.pipelines, 'get', new=self._mock_gets):
fetch_func = partial(
self.test_gocd_client.fetch_pipeline_to_advance,
'manual_verification_edxapp_prod_early_ami_build',
'manual_verification',
None,
)
if exception_expected:
with self.assertRaises(exception_expected):
if current_time:
found_pipeline = fetch_func(current_time)
else:
found_pipeline = fetch_func()
else:
if current_time:
found_pipeline = fetch_func(current_time)
else:
found_pipeline = fetch_func()
self.assertEqual(found_pipeline.name, 'manual_verification_edxapp_prod_early_ami_build')
self.assertEqual(found_pipeline.counter, 148)
INSTANCE_FIND_TEST_DATA = (
(VALID_JOB_TRIGGER_TIME_MS, (False, False), None),
(VALID_JOB_TRIGGER_TIME_MS, (True, True), AdvancementPipelineNotFound),
(VALID_JOB_TRIGGER_TIME_MS, (False, True), AdvancementPipelineAlreadyAdvanced),
(INVALID_JOB_TRIGGER_TIME_MS, (False, False), AdvancementPipelineNotFound)
)
@ddt.data(*INSTANCE_FIND_TEST_DATA)
@ddt.unpack
def test_pipeline_instance_finding_using_specific_time(
self, manual_trigger_time, stage_statuses, exception_expected
):
"""
Verify that the correct pipeline instance is found, given the passed-in time.
"""
self._test_pipeline_instance_finding(
manual_trigger_time,
stage_statuses,
exception_expected,
datetime(2017, 2, 18, 1, 0, 0, tzinfo=tz.gettz('UTC'))
)
@freeze_time("2017-02-18 01:00:00")
@ddt.data(*INSTANCE_FIND_TEST_DATA)
@ddt.unpack
def test_pipeline_instance_finding_using_now(
self, manual_trigger_time, stage_statuses, exception_expected
):
"""
Verify that the correct pipeline instance is found, given the *current* time of now().
"""
self._test_pipeline_instance_finding(
manual_trigger_time,
stage_statuses,
exception_expected
)
def test_pipeline_instance_finding_with_no_data(self):
"""
Verify proper behavior when no history data is returned.
"""
instances = self._build_mocked_gocd_data([])
with patch.object(self.test_gocd_client.client.pipelines, 'full_history', return_value=instances):
with self.assertRaises(AdvancementPipelineNotFound):
self.test_gocd_client.fetch_pipeline_to_advance(
'manual_verification_edxapp_prod_early_ami_build',
'manual_verification'
)
def tearDown(self):
self._instance_map = {}
super(GoCDApiTestCase, self).tearDown()