def test_event_to_complete(cb_one):
"""Test the Freshmaker handler when it receives an event to complete message."""
advisory = Advisory.get_or_create({
'id_': '34727',
'advisory_name': 'RHBA-8018:0600-01'
})[0]
event = FreshmakerEvent.get_or_create({
'id_': '2194',
'state_name': 'BUILDING',
})[0]
event.successful_koji_builds.connect(cb_one)
event.triggered_by_advisory.connect(advisory)
# Load the message to pass to the handler
with open(path.join(message_dir, 'freshmaker', 'event_to_complete.json'), 'r') as f:
msg = json.load(f)
# Make sure the handler can handle the message
assert FreshmakerHandler.can_handle(msg) is True
# Instantiate the handler
handler = FreshmakerHandler(config)
# Run the handler
handler.handle(msg)
event = FreshmakerEvent.nodes.get_or_none(id_='2194')
assert event is not None
assert event.state_name == 'COMPLETE'
assert event.state_reason == '2 of 3 container image(s) failed to rebuild.'
assert event.time_created == datetime(2019, 8, 21, 13, 42, 20, tzinfo=pytz.utc)
assert event.time_done == datetime(2099, 8, 21, 13, 42, 20, tzinfo=pytz.utc)
assert len(event.successful_koji_builds) == 1
def test_one_to_many_node_relationships_failed(client):
"""Tests getting one-to-many relationships of an artifact with wrong relationship name."""
fm_event = FreshmakerEvent.get_or_create({
'event_type_id': 8,
'id_': '1180',
'message_id': 'ID:messaging-devops-broker01.test',
'state': 2,
'state_name': 'COMPLETE',
'state_reason': 'All container images have been rebuilt.',
'time_created': datetime(2019, 8, 21, 13, 42, 20),
'time_done': datetime(2099, 8, 21, 13, 42, 20)
})[0]
cb = ContainerKojiBuild.get_or_create({
'completion_time': datetime(2017, 4, 2, 19, 39, 6),
'creation_time': datetime(2017, 4, 2, 19, 39, 6),
'epoch': '0',
'id_': '710',
'name': 'slf4j_2',
'operator': False,
'release': '4.el7_4_as',
'start_time': datetime(2017, 4, 2, 19, 39, 6),
'state': 1,
'version': '1.7.4'
})[0]
fm_event.successful_koji_builds.connect(cb)
expected = {
'message': 'Please provide a valid relationship name for freshmakerevent with uid 1180',
'status': 400
}
rv = client.get('/api/v1/relationships/freshmakerevent/1180/some_non-existent_relationship')
assert rv.status_code == 400
assert json.loads(rv.data.decode('utf-8')) == expected
def test_build_state_change(mock_koji_cs, mock_getBuild_one):
"""Test the Freshmaker handler when it receives a build state change message."""
mock_koji_session = mock.Mock()
mock_koji_session.getTaskResult.return_value = {'koji_builds': ['710916']}
mock_koji_session.getBuild.return_value = mock_getBuild_one
mock_koji_cs.return_value = mock_koji_session
event = FreshmakerEvent.get_or_create({
'id_': '2094',
'state': 1,
'state_name': 'BUILDING',
'event_type_id': 8,
'message_id': 'ID:messaging.domain.com-42045-1527890187852-9:704208:0:0:1.RHBA-8018:0593-01'
})[0]
# Load the message to pass to the handler
with open(path.join(message_dir, 'freshmaker', 'build_state_change.json'), 'r') as f:
msg = json.load(f)
# Make sure the handler can handle the message
assert FreshmakerHandler.can_handle(msg) is True
# Instantiate the handler
handler = FreshmakerHandler(config)
# Run the handler
handler.handle(msg)
build = ContainerKojiBuild.nodes.get_or_none(id_='710916')
assert build is not None
assert build.completion_time == datetime(2018, 6, 15, 20, 26, 38, tzinfo=pytz.utc)
assert build.creation_time == datetime(2018, 6, 15, 20, 20, 38, tzinfo=pytz.utc)
assert build.epoch == 'epoch'
assert build.extra == '{"container_koji_task_id": 17511743}'
assert build.id_ == '710916'
assert build.name == 'e2e-container-test-product-container'
assert build.original_nvr == 'e2e-container-test-product-container-7.5-133'
assert build.release == '36.1528968216'
assert build.version == '7.4'
assert build.start_time == datetime(2018, 6, 15, 20, 21, 38, tzinfo=pytz.utc)
assert build.state == 1
assert build.triggered_by_freshmaker_event.is_connected(event)
assert event.successful_koji_builds.is_connected(build)
assert len(event.successful_koji_builds) == 1
mock_koji_session.getTaskResult.assert_called_once_with(16735050)
mock_koji_session.getBuild.assert_called_once_with(710916, strict=True)
freshmaker_build = FreshmakerBuild.nodes.get_or_none(id_='1260')
assert freshmaker_build is not None
assert freshmaker_build.id_ == '1260'
assert freshmaker_build.build_id == 16735050
assert freshmaker_build.name == 'e2e-container-test-product-container'
assert freshmaker_build.original_nvr == 'e2e-container-test-product-container-7.5-133'
assert freshmaker_build.rebuilt_nvr == 'e2e-container-test-product-container-7.4-36.1528968216'
assert freshmaker_build.state == 1
assert freshmaker_build.state_name == 'DONE'
assert freshmaker_build.state_reason == 'Built successfully.'
assert freshmaker_build.time_submitted == datetime(2018, 6, 14, 20, 26, 6, tzinfo=pytz.utc)
assert freshmaker_build.type_ == 1
assert freshmaker_build.type_name == 'IMAGE'
assert freshmaker_build.url == 'http://freshmaker.domain.com/api/1/builds/1260'
assert event.requested_builds.is_connected(freshmaker_build)
def test_one_to_many_node_relationships(client, resource, uid, relationship, expected):
"""Tests getting one-to-many relationships of an artifact."""
fm_event = FreshmakerEvent.get_or_create({
'event_type_id': 8,
'id_': '1180',
'message_id': 'ID:messaging-devops-broker01.test',
'state': 2,
'state_name': 'COMPLETE',
'state_reason': 'All container images have been rebuilt.',
'time_created': datetime(2019, 8, 21, 13, 42, 20),
'time_done': datetime(2099, 8, 21, 13, 42, 20)
})[0]
cb = ContainerKojiBuild.get_or_create({
'completion_time': datetime(2017, 4, 2, 19, 39, 6),
'creation_time': datetime(2017, 4, 2, 19, 39, 6),
'epoch': '0',
'id_': '710',
'name': 'slf4j_2',
'operator': False,
'release': '4.el7_4_as',
'start_time': datetime(2017, 4, 2, 19, 39, 6),
'state': 1,
'version': '1.7.4'
})[0]
cb_two = ContainerKojiBuild.get_or_create({
'completion_time': datetime(2018, 4, 2, 19, 39, 6),
'creation_time': datetime(2018, 4, 2, 19, 39, 6),
'epoch': '0',
'id_': '811',
'name': 'some_build',
'operator': False,
'release': '4.el7_4_as',
'start_time': datetime(2018, 4, 2, 19, 39, 6),
'state': 2,
'version': '1.7.5'
})[0]
cb_three = ContainerKojiBuild.get_or_create({
'completion_time': datetime(2018, 4, 2, 19, 39, 6),
'creation_time': datetime(2018, 4, 2, 19, 39, 6),
'epoch': '1',
'id_': '2011',
'name': 'some_other_build',
'operator': False,
'release': '4.el7_6_a',
'start_time': datetime(2018, 4, 2, 19, 39, 6),
'state': 1,
'version': '1.7.6'
})[0]
fm_event.successful_koji_builds.connect(cb)
fm_event.successful_koji_builds.connect(cb_two)
fm_event.successful_koji_builds.connect(cb_three)
rv = client.get('/api/v1/relationships/{0}/{1}/{2}'.format(resource, uid, relationship))
assert rv.status_code == 200
assert json.loads(rv.data.decode('utf-8')) == expected
def event_state_handler(self, msg):
"""
Handle a Freshmaker event state changed message and update Neo4j if necessary.
:param dict msg: a message to be processed
"""
msg_id = msg['body']['msg']['message_id']
if msg['body']['msg'].get('dry_run'):
return
event_params = {
'id_': str(msg['body']['msg']['id']),
'state_name': msg['body']['msg']['state_name'],
'state_reason': msg['body']['msg']['state_reason']
}
if 'time_created' in msg['body']['msg']:
event_params['time_created'] = timestamp_to_datetime(
msg['body']['msg']['time_created'])
if 'time_done' in msg['body']['msg'] and msg['body']['msg'][
'time_done'] is not None:
event_params['time_done'] = timestamp_to_datetime(
msg['body']['msg']['time_done'])
event = FreshmakerEvent.create_or_update(event_params)[0]
advisory_name = msg_id.rsplit('.', 1)[-1]
if advisory_name[0:4] not in ('RHSA', 'RHBA', 'RHEA'):
log.warn(
'Unable to parse the advisory name from the Freshmaker message_id: {0}'
.format(msg_id))
advisory_name = None
advisory = Advisory.get_or_create({
'id_':
msg['body']['msg']['search_key'],
'advisory_name':
advisory_name
})[0]
event.conditional_connect(event.triggered_by_advisory, advisory)
def test_event_to_complete(cb_one):
"""Test the Freshmaker handler when it receives an event to complete message."""
advisory = Advisory.get_or_create({
'id_': '34727',
'advisory_name': 'RHBA-8018:0600-01'
})[0]
event = FreshmakerEvent.get_or_create({
'id_':
'2194',
'state':
1,
'state_name':
'BUILDING',
'event_type_id':
8,
'message_id':
'ID:messaging.domain.com-42045-1527890187852-9:1045742:0:0:1.RHBA-8018:0600-01'
})[0]
event.triggered_container_builds.connect(cb_one)
event.triggered_by_advisory.connect(advisory)
# Load the message to pass to the handler
with open(path.join(message_dir, 'freshmaker', 'event_to_complete.json'),
'r') as f:
msg = json.load(f)
# Make sure the handler can handle the message
assert FreshmakerHandler.can_handle(msg) is True
# Instantiate the handler
handler = FreshmakerHandler(config)
# Run the handler
handler.handle(msg)
event = FreshmakerEvent.nodes.get_or_none(id_='2194')
assert event is not None
assert event.event_type_id == 8
assert event.message_id == \
'ID:messaging.domain.com-42045-1527890187852-9:1045742:0:0:1.RHBA-8018:0600-01'
assert event.state == 2
assert event.state_name == 'COMPLETE'
assert event.state_reason == '2 of 3 container image(s) failed to rebuild.'
assert len(event.triggered_container_builds) == 1
def event_state_handler(self, msg):
"""
Handle a Freshmaker event state changed message and update Neo4j if necessary.
:param dict msg: a message to be processed
"""
msg_id = msg['body']['msg']['message_id']
event = FreshmakerEvent.create_or_update({
'id_':
str(msg['body']['msg']['id']),
'event_type_id':
msg['body']['msg']['event_type_id'],
'message_id':
msg_id,
'state':
msg['body']['msg']['state'],
'state_name':
msg['body']['msg']['state_name'],
'state_reason':
msg['body']['msg']['state_reason']
})[0]
advisory_name = msg_id.rsplit('.', 1)[-1]
if advisory_name[0:4] not in ('RHSA', 'RHBA', 'RHEA'):
log.warn(
'Unable to parse the advisory name from the Freshmaker message_id: {0}'
.format(msg_id))
advisory_name = None
advisory = Advisory.get_or_create({
'id_':
msg['body']['msg']['search_key'],
'advisory_name':
advisory_name
})[0]
event.conditional_connect(event.triggered_by_advisory, advisory)
def test_module_story_node_siblings(client, resource, uid, backward_rel, expected):
"""Tests getting the siblings of an artifact's adjacent node in the module story path."""
bug = BugzillaBug.get_or_create({
'classification': 'Red Hat',
'creation_time': datetime(2017, 4, 2, 19, 39, 6),
'id_': '12345',
'modified_time': datetime(2018, 2, 7, 19, 30, 47),
'priority': 'high',
'product_name': 'Red Hat Enterprise Linux',
'product_version': '7.5',
'resolution': '',
'severity': 'low',
'short_description': 'Some description',
'status': 'VERIFIED',
'target_milestone': 'rc',
'votes': 0
})[0]
commit = DistGitCommit.get_or_create({
'author_date': datetime(2017, 4, 26, 11, 44, 38),
'commit_date': datetime(2017, 4, 26, 11, 44, 38),
'hash_': '8a63adb248ba633e200067e1ad6dc61931727bad',
'log_message': 'Related: #12345 - fix xyz'
})[0]
build = KojiBuild.get_or_create({
'completion_time': datetime(2017, 4, 2, 19, 39, 6),
'creation_time': datetime(2017, 4, 2, 19, 39, 6),
'epoch': '0',
'id_': '2345',
'name': 'slf4j',
'release': '4.el7_4',
'start_time': datetime(2017, 4, 2, 19, 39, 6),
'state': 1,
'version': '1.7.4'
})[0]
build_two = KojiBuild.get_or_create({
'completion_time': datetime(2017, 4, 2, 19, 39, 6),
'creation_time': datetime(2017, 4, 2, 19, 39, 6),
'epoch': '0',
'id_': '3456',
'name': 'slf3j',
'release': '4.el6_3',
'start_time': datetime(2017, 4, 2, 19, 39, 6),
'state': 2,
'version': '1.7.1'
})[0]
module_build = ModuleKojiBuild.get_or_create({
'completion_time': datetime(2017, 4, 2, 19, 39, 6),
'creation_time': datetime(2017, 4, 2, 19, 39, 6),
'epoch': '0',
'id_': '2345',
'name': '389-ds',
'context': 'a2037af3',
'release': '20180805121332.a2037af3',
'start_time': datetime(2017, 4, 2, 19, 39, 6),
'mbs_id': 1338,
'module_name': '389-ds',
'module_version': '20180805121332',
'module_stream': '1.4'
})[0]
advisory = Advisory.get_or_create({
'actual_ship_date': datetime(2017, 8, 1, 15, 43, 51),
'advisory_name': 'RHBA-2017:2251-02',
'content_types': ['docker'],
'created_at': datetime(2017, 4, 3, 14, 47, 23),
'id_': '27825',
'issue_date': datetime(2017, 8, 1, 5, 59, 34),
'product_name': 'Red Hat Enterprise Linux',
'product_short_name': 'RHEL',
'security_impact': 'None',
'state': 'SHIPPED_LIVE',
'status_time': datetime(2017, 8, 1, 15, 43, 51),
'synopsis': 'cifs-utils bug fix update',
'update_date': datetime(2017, 8, 1, 7, 16)
})[0]
fm_event = FreshmakerEvent.get_or_create({
'event_type_id': 8,
'id_': '1180',
'message_id': 'ID:messaging-devops-broker01.test',
'state': 2,
'state_name': 'COMPLETE',
'state_reason': 'All container images have been rebuilt.',
'time_created': datetime(2019, 8, 21, 13, 42, 20),
'time_done': datetime(2099, 8, 21, 13, 42, 20)
})[0]
cb = ContainerKojiBuild.get_or_create({
'completion_time': datetime(2017, 4, 2, 19, 39, 6),
'creation_time': datetime(2017, 4, 2, 19, 39, 6),
'epoch': '0',
'id_': '710',
'name': 'slf4j_2',
'release': '4.el7_4_as',
'start_time': datetime(2017, 4, 2, 19, 39, 6),
'state': 1,
'version': '1.7.4'
})[0]
cb_two = ContainerKojiBuild.get_or_create({
'completion_time': datetime(2018, 4, 2, 19, 39, 6),
'creation_time': datetime(2018, 4, 2, 19, 39, 6),
'epoch': '0',
'id_': '811',
'name': 'some_build',
'release': '4.el7_4_as',
'start_time': datetime(2018, 4, 2, 19, 39, 6),
'state': 2,
'version': '1.7.5'
})[0]
ca = ContainerAdvisory.get_or_create({
'actual_ship_date': datetime(2017, 8, 1, 15, 43, 51),
'advisory_name': 'RHBA-2017:2251-03',
'content_types': ['docker'],
'created_at': datetime(2017, 4, 3, 14, 47, 23),
'id_': '12327',
'issue_date': datetime(2017, 8, 1, 5, 59, 34),
'product_name': 'Red Hat Enterprise Linux',
'product_short_name': 'RHEL',
'security_impact': 'None',
'state': 'SHIPPED_LIVE',
'status_time': datetime(2017, 8, 1, 15, 43, 51),
'synopsis': 'cifs-utils bug fix update',
'update_date': datetime(2017, 8, 1, 7, 16)
})[0]
commit.resolved_bugs.connect(bug)
commit.koji_builds.connect(build)
build.advisories.connect(advisory)
build_two.advisories.connect(advisory)
fm_event.triggered_by_advisory.connect(advisory)
fm_event.successful_koji_builds.connect(cb)
fm_event.successful_koji_builds.connect(cb_two)
ca.attached_builds.connect(cb)
ca.attached_builds.connect(cb_two)
module_build.components.connect(build)
module_build.components.connect(build_two)
module_build.advisories.connect(advisory)
url = '/api/v1/siblings/{0}/{1}?story_type=module'.format(resource, uid)
if backward_rel:
url = '{0}&backward_rel=true'.format(url)
rv = client.get(url)
assert rv.status_code == 200
assert json.loads(rv.data.decode('utf-8')) == expected
def test_all_stories(client, resource, uid, expected):
"""Test getting all unique stories for an artifact."""
bug = BugzillaBug.get_or_create({
'creation_time': datetime(2017, 4, 2, 19, 39, 6),
'id_': '12345',
'modified_time': datetime(2018, 2, 7, 19, 30, 47),
'priority': 'high',
'product_name': 'Red Hat Enterprise Linux',
'product_version': '7.5',
'resolution': '',
'severity': 'low',
'short_description': 'Some description',
'status': 'VERIFIED',
'target_milestone': 'rc',
})[0]
bug_two = BugzillaBug.get_or_create({
'creation_time': datetime(2017, 4, 1, 17, 41, 4),
'severity': 'medium',
'short_description': 'some description',
'product_version': '7.2',
'priority': 'unspecified',
'product_name': 'Red Hat Enterprise Linux 7',
'resolution': 'DUPLICATE',
'target_milestone': 'rc',
'modified_time': datetime(2018, 3, 14, 5, 53, 19),
'id_': '1245',
'status': 'CLOSED'
})[0]
commit = DistGitCommit.get_or_create({
'author_date': datetime(2017, 4, 26, 11, 44, 38),
'commit_date': datetime(2018, 5, 2, 10, 36, 47),
'hash_': '8a63adb248ba633e200067e1ad6dc61931727bad',
'log_message': 'Related: #12345 - fix xyz'
})[0]
commit_two = DistGitCommit.get_or_create({
'commit_date': datetime(2018, 3, 14, 5, 52, 19),
'author_date': datetime(2018, 3, 14, 5, 53, 25),
'log_message': 'Repo creation',
'hash_': 'f4dfc64c10a90492303e4f14ad3549a1a2b13575'
})[0]
build = KojiBuild.get_or_create({
'completion_time': datetime(2018, 6, 2, 10, 55, 47),
'creation_time': datetime(2018, 6, 2, 10, 36, 47),
'epoch': '0',
'id_': '2345',
'name': 'slf4j',
'release': '4.el7_4',
'start_time': datetime(2018, 6, 2, 10, 36, 47),
'state': 1,
'version': '1.7.4'
})[0]
advisory = Advisory.get_or_create({
'actual_ship_date': datetime(2017, 8, 1, 15, 43, 51),
'advisory_name': 'RHBA-2017:2251-02',
'created_at': datetime(2018, 6, 13, 10, 36, 47),
'id_': '27825',
'issue_date': datetime(2017, 8, 1, 5, 59, 34),
'product_name': 'Red Hat Enterprise Linux',
'security_impact': 'None',
'state': 'SHIPPED_LIVE',
'status_time': datetime(2017, 8, 1, 15, 43, 51),
'synopsis': 'cifs-utils bug fix update',
'update_date': datetime(2017, 8, 1, 7, 16)
})[0]
advisory_two = Advisory.get_or_create({
'security_impact': 'None',
'created_at': datetime(2018, 4, 21, 19, 36, 47),
'synopsis': 'This is a synopsis of a test advisory.',
'product_name': 'Release End2End Test',
'update_date': datetime(2018, 4, 21, 19, 36, 47),
'advisory_name': 'RHBA-2017:27760-01',
'issue_date': datetime(2018, 3, 14, 5, 53, 25),
'status_time': datetime(2018, 3, 14, 7, 53, 25),
'state': 'DROPPED_NO_SHIP',
'id_': '123456'
})[0]
fm_event = FreshmakerEvent.get_or_create({
'id_': '1180',
'state_name': 'COMPLETE',
'state_reason': 'All container images have been rebuilt.',
'time_created': datetime(2018, 8, 13, 10, 36, 47),
'time_done': datetime(2018, 8, 13, 12, 45, 47)
})[0]
cb = ContainerKojiBuild.get_or_create({
'completion_time': datetime(2017, 4, 2, 19, 39, 6),
'creation_time': datetime(2017, 4, 2, 19, 39, 6),
'epoch': '0',
'id_': '710',
'name': 'slf4j_2',
'release': '4.el7_4_as',
'start_time': datetime(2017, 4, 2, 19, 39, 6),
'state': 1,
'version': '1.7.4'
})[0]
cb_two = ContainerKojiBuild.get_or_create({
'completion_time': datetime(2018, 8, 17, 13, 55, 47),
'creation_time': datetime(2018, 8, 17, 8, 32, 47),
'epoch': '0',
'id_': '811',
'name': 'some_build',
'release': '4.el7_4_as',
'start_time': datetime(2018, 8, 17, 8, 32, 47),
'state': 2,
'version': '1.7.5'
})[0]
# Longest story
commit.resolved_bugs.connect(bug)
commit.koji_builds.connect(build)
build.advisories.connect(advisory, {'time_attached': datetime(2018, 6, 13, 10, 36, 47)})
fm_event.triggered_by_advisory.connect(advisory)
fm_event.successful_koji_builds.connect(cb)
fm_event.successful_koji_builds.connect(cb_two)
# Unique partial stories
commit_two.resolved_bugs.connect(bug_two)
commit_two.koji_builds.connect(build)
build.advisories.connect(advisory_two, {'time_attached': datetime(2018, 4, 21, 19, 36, 47)})
rv = client.get('/api/v1/allstories/{0}/{1}'.format(resource, uid))
assert rv.status_code == 200
assert json.loads(rv.data.decode('utf-8')) == expected
def query_api_and_update_neo4j(self):
"""
Scrape the Freshmaker API and upload the data to Neo4j.
:param str start_date: a datetime to start scraping data from
"""
# Initialize session and url
session = retry_session()
fm_url = self.freshmaker_url
while True:
log.debug('Querying {0}'.format(fm_url))
try:
rv_json = session.get(fm_url, timeout=60).json()
except ConnectionError:
# TODO: Remove this once FACTORY-3955 is resolved
log.error(
'The connection to Freshmaker at %s failed. Skipping the rest of the scraper.',
fm_url,
)
break
for fm_event in rv_json['items']:
try:
int(fm_event['search_key'])
except ValueError:
# Skip Freshmaker Events that don't have the search_key as the Advisory ID
continue
log.debug('Creating FreshmakerEvent {0}'.format(fm_event['id']))
event_params = dict(
id_=fm_event['id'],
event_type_id=fm_event['event_type_id'],
message_id=fm_event['message_id'],
state=fm_event['state'],
state_name=fm_event['state_name'],
state_reason=fm_event['state_reason'],
url=fm_event['url']
)
if fm_event.get('time_created'):
event_params['time_created'] = timestamp_to_datetime(fm_event['time_created'])
if fm_event.get('time_done'):
event_params['time_done'] = timestamp_to_datetime(fm_event['time_created'])
event = FreshmakerEvent.create_or_update(event_params)[0]
log.debug('Creating Advisory {0}'.format(fm_event['search_key']))
advisory = Advisory.get_or_create(dict(
id_=fm_event['search_key']
))[0]
event.conditional_connect(event.triggered_by_advisory, advisory)
for build_dict in fm_event['builds']:
# To handle a faulty container build in Freshmaker
if build_dict['build_id'] and int(build_dict['build_id']) < 0:
continue
log.debug('Creating FreshmakerBuild {0}'.format(build_dict['build_id']))
fb_params = dict(
id_=build_dict['id'],
dep_on=build_dict['dep_on'],
name=build_dict['name'],
original_nvr=build_dict['original_nvr'],
rebuilt_nvr=build_dict['rebuilt_nvr'],
state=build_dict['state'],
state_name=build_dict['state_name'],
state_reason=build_dict['state_reason'],
time_submitted=timestamp_to_datetime(build_dict['time_submitted']),
type_=build_dict['type'],
type_name=build_dict['type_name'],
url=build_dict['url']
)
if build_dict['time_completed']:
fb_params['time_completed'] = timestamp_to_datetime(
build_dict['time_completed'])
if build_dict['build_id']:
fb_params['build_id'] = build_dict['build_id']
fb = FreshmakerBuild.create_or_update(fb_params)[0]
event.requested_builds.connect(fb)
# The build ID obtained from Freshmaker API is actually a Koji task ID
task_result = None
if build_dict['build_id']:
task_result = self.get_koji_task_result(build_dict['build_id'])
if not task_result:
continue
# Extract the build ID from a task result
xml_root = ET.fromstring(task_result)
# TODO: Change this if a task can trigger multiple builds
try:
build_id = xml_root.find(".//*[name='koji_builds'].//string").text
except AttributeError:
build_id = None
if not build_id:
continue
log.debug('Creating ContainerKojiBuild {0}'.format(build_id))
build_params = {
'id_': build_id,
'original_nvr': build_dict['original_nvr']
}
try:
build = ContainerKojiBuild.create_or_update(build_params)[0]
except neomodel.exceptions.ConstraintValidationFailed:
# This must have errantly been created as a KojiBuild instead of a
# ContainerKojiBuild, so let's fix that.
build = KojiBuild.nodes.get_or_none(id_=build_id)
if not build:
# If there was a constraint validation failure and the build isn't just
# the wrong label, then we can't recover.
raise
build.add_label(ContainerKojiBuild.__label__)
build = ContainerKojiBuild.create_or_update(build_params)[0]
event.successful_koji_builds.connect(build)
if rv_json['meta'].get('next'):
fm_url = rv_json['meta']['next']
else:
break
def test_get_resources(client, resource, uid, expected):
"""Test getting a resource from Neo4j with its relationships."""
tbrady = User.get_or_create({
'email': '*****@*****.**',
'username': '******'
})[0]
mprahl = User.get_or_create({
'email': '*****@*****.**',
'username': '******'
})[0]
jsmith = User.get_or_create({
'email': '*****@*****.**',
'username': '******'
})[0]
commit = DistGitCommit.get_or_create({
'author_date': datetime(2017, 4, 26, 11, 44, 38),
'commit_date': datetime(2017, 4, 26, 11, 44, 38),
'hash_': '8a63adb248ba633e200067e1ad6dc61931727bad',
'log_message': 'Related: #12345 - fix xyz'
})[0]
commit_two = DistGitCommit.get_or_create({
'author_date': datetime(2017, 4, 27, 11, 44, 38),
'commit_date': datetime(2017, 4, 27, 11, 44, 38),
'hash_': '1263adb248ba633e205067e1ad6dc61931727c2d',
'log_message': 'Related: #12345 - fix xz'
})[0]
commit_three = DistGitCommit.get_or_create({
'author_date': datetime(2017, 4, 27, 11, 44, 38),
'commit_date': datetime(2017, 4, 27, 11, 44, 38),
'hash_': '5663adb248ba633e205067e1ad6dc61931727123',
'log_message': 'Revert: #12345'
})[0]
bug = BugzillaBug.get_or_create({
'classification': 'Red Hat',
'creation_time': datetime(2017, 4, 2, 19, 39, 6),
'id_': '12345',
'modified_time': datetime(2018, 2, 7, 19, 30, 47),
'priority': 'high',
'product_name': 'Red Hat Enterprise Linux',
'product_version': '7.5',
'resolution': '',
'severity': 'low',
'short_description': 'Some description',
'status': 'VERIFIED',
'target_milestone': 'rc',
'votes': 0
})[0]
bug_two = BugzillaBug.get_or_create({
'classification': 'Red Hat',
'creation_time': datetime(2017, 4, 2, 19, 39, 6),
'id_': '67890',
'modified_time': datetime(2018, 2, 7, 19, 30, 47),
'priority': 'medium',
'product_name': 'Red Hat Enterprise Linux',
'product_version': '7.3',
'resolution': '',
'severity': 'low',
'short_description': 'Some description',
'status': 'VERIFIED',
'target_milestone': 'rc',
'votes': 0
})[0]
bug_three = BugzillaBug.get_or_create({
'classification': 'Red Hat',
'creation_time': datetime(2017, 4, 2, 19, 39, 6),
'id_': '272895',
'modified_time': datetime(2018, 2, 7, 19, 30, 47),
'priority': 'low',
'product_name': 'Satellite',
'product_version': '3',
'resolution': '',
'severity': 'medium',
'short_description': 'Some description',
'status': 'VERIFIED',
'target_milestone': 'rc',
'votes': 0
})[0]
repo = DistGitRepo.get_or_create({
'name': 'some_repo',
'namespace': 'some_namespace',
})[0]
branch = DistGitBranch.get_or_create({
'name': 'some_branch_name',
'repo_name': 'some_repo_name',
'repo_namespace': 'some_repo_namespace'
})[0]
build = KojiBuild.get_or_create({
'completion_time': datetime(2017, 4, 2, 19, 39, 6),
'creation_time': datetime(2017, 4, 2, 19, 39, 6),
'epoch': '0',
'id_': '2345',
'name': 'slf4j',
'release': '4.el7_4',
'start_time': datetime(2017, 4, 2, 19, 39, 6),
'state': 1,
'version': '1.7.4'
})[0]
tag = KojiTag.get_or_create({
'id_': '2702',
'name': 'some_active_tag'
})[0]
advisory = Advisory.get_or_create({
'actual_ship_date': datetime(2017, 8, 1, 15, 43, 51),
'advisory_name': 'RHBA-2017:2251-02',
'content_types': ['docker'],
'created_at': datetime(2017, 4, 3, 14, 47, 23),
'id_': '27825',
'issue_date': datetime(2017, 8, 1, 5, 59, 34),
'product_name': 'Red Hat Enterprise Linux',
'product_short_name': 'RHEL',
'security_impact': 'None',
'state': 'SHIPPED_LIVE',
'status_time': datetime(2017, 8, 1, 15, 43, 51),
'synopsis': 'cifs-utils bug fix update',
'update_date': datetime(2017, 8, 1, 7, 16)
})[0]
fm_event = FreshmakerEvent.get_or_create({
'event_type_id': 8,
'id_': '1180',
'message_id': 'ID:messaging-devops-broker01.test',
'state': 2,
'state_name': 'COMPLETE',
'state_reason': 'All container images have been rebuilt',
'time_created': datetime(2019, 8, 21, 13, 42, 20),
'time_done': datetime(2099, 8, 21, 13, 42, 20)
})[0]
fm_build = FreshmakerBuild.get_or_create({
'id_': 398,
'build_id': 15639305,
'dep_on': "jboss-eap-7-eap70-openshift-docker",
'name': "metrics-hawkular-metrics-docker",
'original_nvr': "metrics-hawkular-metrics-docker-v3.7.23-10",
'rebuilt_nvr': "metrics-hawkular-metrics-docker-v3.7.23-10.1522094767",
'state': 1,
'state_name': "DONE",
'state_reason': "Built successfully.",
'time_completed': datetime(2017, 4, 2, 19, 39, 6),
'time_submitted': datetime(2017, 4, 2, 19, 39, 6),
'type_': 1,
'type_name': "IMAGE",
'url': "/api/1/builds/398"
})[0]
cb = ContainerKojiBuild.get_or_create({
'completion_time': datetime(2017, 4, 2, 19, 39, 6),
'creation_time': datetime(2017, 4, 2, 19, 39, 6),
'epoch': '0',
'id_': '710',
'name': 'slf4j_2',
'release': '4.el7_4_as',
'start_time': datetime(2017, 4, 2, 19, 39, 6),
'state': 1,
'version': '1.7.4'
})[0]
if resource == 'bugzillabug':
bug.assignee.connect(mprahl)
bug.qa_contact.connect(jsmith)
bug.reporter.connect(tbrady)
commit.resolved_bugs.connect(bug)
commit_two.resolved_bugs.connect(bug)
commit_three.reverted_bugs.connect(bug)
advisory.attached_bugs.connect(bug)
if resource == 'distgitcommit':
commit.author.connect(tbrady)
commit.parent.connect(commit_two)
commit_three.parent.connect(commit)
commit.related_bugs.connect(bug)
commit.related_bugs.connect(bug_three)
commit.reverted_bugs.connect(bug_two)
repo.commits.connect(commit)
branch.commits.connect(commit)
commit.resolved_bugs.connect(bug)
commit.resolved_bugs.connect(bug_two)
if resource == 'kojibuild':
build.owner.connect(mprahl)
build.commit.connect(commit_two)
tag.builds.connect(build)
if resource == 'advisory':
advisory.assigned_to.connect(mprahl)
advisory.reporter.connect(jsmith)
advisory.attached_builds.connect(build)
advisory.attached_bugs.connect(bug)
if resource == 'freshmakerevent':
fm_event.triggered_by_advisory.connect(advisory)
fm_event.successful_koji_builds.connect(cb)
fm_event.requested_builds.connect(fm_build)
if resource == 'containerbuild':
fm_event.successful_koji_builds.connect(cb)
rv = client.get('/api/v1/{0}/{1}'.format(resource, uid))
assert rv.status_code == 200
assert json.loads(rv.data.decode('utf-8')) == expected
def test_get_recent_nodes(client):
"""Test the get_recent_nodes function."""
id_dict = {
FreshmakerEvent.__label__: 'id',
BugzillaBug.__label__: 'id',
DistGitCommit.__label__: 'hash',
KojiBuild.__label__: 'id',
Advisory.__label__: 'id'
}
timestamp_dict = {
FreshmakerEvent.__label__: 'time_created',
BugzillaBug.__label__: 'modified_time',
DistGitCommit.__label__: 'commit_date',
KojiBuild.__label__: 'completion_time',
Advisory.__label__: 'update_date'
}
expected = {
'data': {
'Advisory': [{
'advisory_name': 'RHBA-2017:27760-01',
'update_date': '2017-05-30T11:44:38Z',
'issue_date': None,
'created_at': None,
'state': None,
'product_name': None,
'security_sla': None,
'synopsis': None,
'security_impact': None,
'status_time': None,
'actual_ship_date': None,
'release_date': None,
'id': '66666',
'display_name': 'RHBA-2017:27760-01',
'resource_type': 'Advisory',
'assigned_to': None,
'attached_bugs': [],
'attached_builds': [],
'triggered_freshmaker_event': [],
'reporter': None
}],
'DistGitCommit': [{
'log_message': None,
'author': None,
'author_date': None,
'koji_builds': [],
'hash': '55555',
'commit_date': '2017-05-02T11:44:38Z',
'display_name': 'commit #55555',
'resource_type': 'DistGitCommit',
'related_bugs': [],
'repos': [],
'resolved_bugs': [],
'reverted_bugs': []
}],
'FreshmakerEvent': [{
'state_reason': None,
'state_name': None,
'time_created': '2017-05-30T11:44:38Z',
'time_done': None,
'id': '77777',
'display_name': 'Freshmaker event 77777',
'resource_type': 'FreshmakerEvent',
'requested_builds': [],
'successful_koji_builds': [],
'triggered_by_advisory': None,
}],
'KojiBuild': [{
'name': 'slf4j',
'start_time': None,
'creation_time': None,
'state': None,
'completion_time': '2017-05-27T11:44:38Z',
'epoch': None,
'version': '1.7.4',
'release': '4.el7_4',
'id': '44444',
'display_name': 'slf4j-1.7.4-4.el7_4',
'resource_type': 'KojiBuild',
'module_builds': [],
'owner': None,
'commit': None,
'advisories': []
}],
'BugzillaBug': [{
'status': None,
'severity': None,
'resolution': None,
'product_version': None,
'creation_time': None,
'modified_time': '2017-06-26T11:44:38Z',
'product_name': None,
'priority': None,
'short_description': None,
'target_milestone': None,
'id': '22222',
'display_name': 'RHBZ#22222',
'resource_type': 'BugzillaBug',
'assignee': None,
'attached_advisories': [],
'qa_contact': None,
'related_by_commits': [],
'reporter': None,
'resolved_by_commits': [],
'reverted_by_commits': []
}, {
'status': None,
'severity': None,
'resolution': None,
'product_version': None,
'creation_time': None,
'modified_time': '2017-05-26T11:44:38Z',
'product_name': None,
'priority': None,
'short_description': None,
'target_milestone': None,
'id': '33333',
'display_name': 'RHBZ#33333',
'resource_type': 'BugzillaBug',
'assignee': None,
'attached_advisories': [],
'qa_contact': None,
'related_by_commits': [],
'reporter': None,
'resolved_by_commits': [],
'reverted_by_commits': []
}, {
'status': None,
'severity': None,
'resolution': None,
'product_version': None,
'creation_time': None,
'modified_time': '2017-04-26T11:44:38Z',
'product_name': None,
'priority': None,
'short_description': None,
'target_milestone': None,
'id': '11111',
'display_name': 'RHBZ#11111',
'resource_type': 'BugzillaBug',
'assignee': None,
'attached_advisories': [],
'qa_contact': None,
'related_by_commits': [],
'reporter': None,
'resolved_by_commits': [],
'reverted_by_commits': []
}]
},
'metadata': {
'id_keys': id_dict,
'timestamp_keys': timestamp_dict
}
}
BugzillaBug.get_or_create({
'id_':
'11111',
'modified_time':
datetime(2017, 4, 26, 11, 44, 38)
})
BugzillaBug.get_or_create({
'id_':
'22222',
'modified_time':
datetime(2017, 6, 26, 11, 44, 38)
})
BugzillaBug.get_or_create({
'id_':
'33333',
'modified_time':
datetime(2017, 5, 26, 11, 44, 38)
})
KojiBuild.get_or_create({
'id_':
'44444',
'completion_time':
datetime(2017, 5, 27, 11, 44, 38),
'name':
'slf4j',
'version':
'1.7.4',
'release':
'4.el7_4'
})
DistGitCommit.get_or_create({
'hash_': '55555',
'commit_date': datetime(2017, 5, 2, 11, 44, 38)
})
Advisory.get_or_create({
'id_': '66666',
'update_date': datetime(2017, 5, 30, 11, 44, 38),
'advisory_name': 'RHBA-2017:27760-01'
})
FreshmakerEvent.get_or_create({
'id_':
'77777',
'time_created':
datetime(2017, 5, 30, 11, 44, 38),
})
rv = client.get('/api/v1/recents')
assert rv.status_code == 200
assert json.loads(rv.data.decode('utf-8')) == expected
def query_api_and_update_neo4j(self):
"""
Scrape the Freshmaker API and upload the data to Neo4j.
:param str start_date: a datetime to start scraping data from
"""
# Initialize session and url
session = retry_session()
fm_url = self.freshmaker_url
while True:
log.debug('Querying {0}'.format(fm_url))
rv_json = session.get(fm_url, timeout=15).json()
for fm_event in rv_json['items']:
try:
int(fm_event['search_key'])
except ValueError:
# Skip Freshmaker Events that don't have the search_key as the Advisory ID
continue
event = FreshmakerEvent.create_or_update(dict(
id_=fm_event['id'],
event_type_id=fm_event['event_type_id'],
message_id=fm_event['message_id'],
state=fm_event['state'],
state_name=fm_event['state_name'],
state_reason=fm_event['state_reason'],
url=fm_event['url']
))[0]
advisory = Advisory.get_or_create(dict(
id_=fm_event['search_key']
))[0]
event.conditional_connect(event.triggered_by_advisory, advisory)
for build_dict in fm_event['builds']:
# To handle a faulty container build in Freshmaker
if not build_dict['build_id'] or int(build_dict['build_id']) < 0:
continue
# The build ID obtained from Freshmaker API is actually a Koji task ID
task_result = self.get_koji_task_result(build_dict['build_id'])
if not task_result:
continue
# Extract the build ID from a task result
xml_root = ET.fromstring(task_result)
# TODO: Change this if a task can trigger multiple builds
try:
build_id = xml_root.find(".//*[name='koji_builds'].//string").text
except AttributeError:
build_id = None
if build_id:
build = ContainerKojiBuild.get_or_create(dict(
id_=build_id,
original_nvr=build_dict['original_nvr']
))[0]
event.triggered_container_builds.connect(build)
if rv_json['meta'].get('next'):
fm_url = rv_json['meta']['next']
else:
break
def test_get_recent_nodes():
"""Test the get_recent_nodes function."""
BugzillaBug.get_or_create({
'id_':
'11111',
'modified_time':
datetime(2017, 4, 26, 11, 44, 38)
})
BugzillaBug.get_or_create({
'id_':
'22222',
'modified_time':
datetime(2017, 6, 26, 11, 44, 38)
})
BugzillaBug.get_or_create({
'id_':
'33333',
'modified_time':
datetime(2017, 5, 26, 11, 44, 38)
})
KojiBuild.get_or_create({
'id_':
'44444',
'completion_time':
datetime(2017, 5, 27, 11, 44, 38)
})
DistGitCommit.get_or_create({
'hash_': '55555',
'commit_date': datetime(2017, 5, 2, 11, 44, 38)
})
Advisory.get_or_create({
'id_': '66666',
'update_date': datetime(2017, 5, 30, 11, 44, 38)
})
Advisory.get_or_create({'id_': '66666', 'update_date': None})
FreshmakerEvent.get_or_create({
'id_':
'77777',
'time_created':
datetime(2017, 5, 30, 11, 44, 38),
})
nodes, meta = estuary.utils.recents.get_recent_nodes()
assert nodes['Advisory'][0]['id'] == '66666'
assert nodes['DistGitCommit'][0]['hash'] == '55555'
assert nodes['FreshmakerEvent'][0]['id'] == '77777'
assert nodes['KojiBuild'][0]['id'] == '44444'
assert nodes['BugzillaBug'][0]['id'] == '22222'
assert nodes['BugzillaBug'][1]['id'] == '33333'
assert nodes['BugzillaBug'][2]['id'] == '11111'
id_dict = {
FreshmakerEvent.__label__: 'id',
BugzillaBug.__label__: 'id',
DistGitCommit.__label__: 'hash',
KojiBuild.__label__: 'id',
Advisory.__label__: 'id'
}
timestamp_dict = {
FreshmakerEvent.__label__: 'time_created',
BugzillaBug.__label__: 'modified_time',
DistGitCommit.__label__: 'commit_date',
KojiBuild.__label__: 'completion_time',
Advisory.__label__: 'update_date'
}
assert meta['id_keys'] == id_dict
assert meta['timestamp_keys'] == timestamp_dict
def test_get_stories(client, resource, uid, expected):
"""Test getting a resource story from Neo4j with its relationships."""
commit = DistGitCommit.get_or_create({
'author_date':
datetime(2017, 4, 26, 11, 44, 38),
'commit_date':
datetime(2017, 4, 26, 11, 44, 38),
'hash_':
'8a63adb248ba633e200067e1ad6dc61931727bad',
'log_message':
'Related: #12345 - fix xyz'
})[0]
advisory = Advisory.get_or_create({
'actual_ship_date':
datetime(2017, 8, 1, 15, 43, 51),
'advisory_name':
'RHBA-2017:2251-02',
'content_types': ['docker'],
'created_at':
datetime(2017, 4, 3, 14, 47, 23),
'id_':
'27825',
'issue_date':
datetime(2017, 8, 1, 5, 59, 34),
'product_name':
'Red Hat Enterprise Linux',
'product_short_name':
'RHEL',
'security_impact':
'None',
'state':
'SHIPPED_LIVE',
'status_time':
datetime(2017, 8, 1, 15, 43, 51),
'synopsis':
'cifs-utils bug fix update',
'type_':
'RHBA',
'update_date':
datetime(2017, 8, 1, 7, 16),
'updated_at':
datetime(2017, 8, 1, 15, 43, 51)
})[0]
bug = BugzillaBug.get_or_create({
'classification':
'Red Hat',
'creation_time':
datetime(2017, 4, 2, 19, 39, 6),
'id_':
'12345',
'modified_time':
datetime(2018, 2, 7, 19, 30, 47),
'priority':
'high',
'product_name':
'Red Hat Enterprise Linux',
'product_version':
'7.5',
'resolution':
'',
'severity':
'low',
'short_description':
'Some description',
'status':
'VERIFIED',
'target_milestone':
'rc',
'votes':
0
})[0]
bug_two = BugzillaBug.get_or_create({
'classification':
'Red Hat',
'creation_time':
datetime(2017, 4, 2, 6, 43, 58),
'id_':
'5555',
'modified_time':
datetime(2017, 12, 5, 10, 12, 47),
'priority':
'unspecified',
'product_name':
'Red Hat CloudForms Management Engine',
'product_version':
'5.7.0',
'resolution':
'WORKSFORME',
'severity':
'unspecified',
'short_description':
'Fail to delete OSP tenant by CFME',
'status':
'CLOSED',
'target_milestone':
'GA',
'votes':
0
})[0]
build = KojiBuild.get_or_create({
'completion_time':
datetime(2017, 4, 2, 19, 39, 6),
'creation_time':
datetime(2017, 4, 2, 19, 39, 6),
'epoch':
'0',
'id_':
'2345',
'name':
'slf4j',
'release':
'4.el7_4',
'start_time':
datetime(2017, 4, 2, 19, 39, 6),
'state':
1,
'version':
'1.7.4'
})[0]
fm_event = FreshmakerEvent.get_or_create({
'event_type_id': 8,
'id_': '1180',
'message_id': 'ID:messaging-devops-broker01.test',
'state': 2,
'state_name': 'COMPLETE',
'state_reason': 'All container images have been rebuilt.',
'url': '/api/1/events/1180'
})[0]
cb = ContainerKojiBuild.get_or_create({
'completion_time':
datetime(2017, 4, 2, 19, 39, 6),
'creation_time':
datetime(2017, 4, 2, 19, 39, 6),
'epoch':
'0',
'id_':
'710',
'name':
'slf4j_2',
'release':
'4.el7_4_as',
'start_time':
datetime(2017, 4, 2, 19, 39, 6),
'state':
1,
'version':
'1.7.4'
})[0]
commit.resolved_bugs.connect(bug_two)
commit.resolved_bugs.connect(bug)
commit.koji_builds.connect(build)
build.advisories.connect(advisory)
advisory.attached_builds.connect(build)
fm_event.triggered_by_advisory.connect(advisory)
fm_event.triggered_container_builds.connect(cb)
rv = client.get('/api/v1/story/{0}/{1}'.format(resource, uid))
assert rv.status_code == 200
assert json.loads(rv.data.decode('utf-8')) == expected
