def event_state_handler(self, msg):
"""
Handle a Freshmaker event state changed message and update Neo4j if necessary.
:param dict msg: a message to be processed
"""
msg_id = msg['body']['msg']['message_id']
if msg['body']['msg'].get('dry_run'):
return
event_params = {
'id_': str(msg['body']['msg']['id']),
'state_name': msg['body']['msg']['state_name'],
'state_reason': msg['body']['msg']['state_reason']
}
if 'time_created' in msg['body']['msg']:
event_params['time_created'] = timestamp_to_datetime(
msg['body']['msg']['time_created'])
if 'time_done' in msg['body']['msg'] and msg['body']['msg'][
'time_done'] is not None:
event_params['time_done'] = timestamp_to_datetime(
msg['body']['msg']['time_done'])
event = FreshmakerEvent.create_or_update(event_params)[0]
advisory_name = msg_id.rsplit('.', 1)[-1]
if advisory_name[0:4] not in ('RHSA', 'RHBA', 'RHEA'):
log.warn(
'Unable to parse the advisory name from the Freshmaker message_id: {0}'
.format(msg_id))
advisory_name = None
advisory = Advisory.get_or_create({
'id_':
msg['body']['msg']['search_key'],
'advisory_name':
advisory_name
})[0]
event.conditional_connect(event.triggered_by_advisory, advisory)
def event_state_handler(self, msg):
"""
Handle a Freshmaker event state changed message and update Neo4j if necessary.
:param dict msg: a message to be processed
"""
msg_id = msg['body']['msg']['message_id']
event = FreshmakerEvent.create_or_update({
'id_':
str(msg['body']['msg']['id']),
'event_type_id':
msg['body']['msg']['event_type_id'],
'message_id':
msg_id,
'state':
msg['body']['msg']['state'],
'state_name':
msg['body']['msg']['state_name'],
'state_reason':
msg['body']['msg']['state_reason']
})[0]
advisory_name = msg_id.rsplit('.', 1)[-1]
if advisory_name[0:4] not in ('RHSA', 'RHBA', 'RHEA'):
log.warn(
'Unable to parse the advisory name from the Freshmaker message_id: {0}'
.format(msg_id))
advisory_name = None
advisory = Advisory.get_or_create({
'id_':
msg['body']['msg']['search_key'],
'advisory_name':
advisory_name
})[0]
event.conditional_connect(event.triggered_by_advisory, advisory)
def query_api_and_update_neo4j(self):
"""
Scrape the Freshmaker API and upload the data to Neo4j.
:param str start_date: a datetime to start scraping data from
"""
# Initialize session and url
session = retry_session()
fm_url = self.freshmaker_url
while True:
log.debug('Querying {0}'.format(fm_url))
try:
rv_json = session.get(fm_url, timeout=60).json()
except ConnectionError:
# TODO: Remove this once FACTORY-3955 is resolved
log.error(
'The connection to Freshmaker at %s failed. Skipping the rest of the scraper.',
fm_url,
)
break
for fm_event in rv_json['items']:
try:
int(fm_event['search_key'])
except ValueError:
# Skip Freshmaker Events that don't have the search_key as the Advisory ID
continue
log.debug('Creating FreshmakerEvent {0}'.format(fm_event['id']))
event_params = dict(
id_=fm_event['id'],
event_type_id=fm_event['event_type_id'],
message_id=fm_event['message_id'],
state=fm_event['state'],
state_name=fm_event['state_name'],
state_reason=fm_event['state_reason'],
url=fm_event['url']
)
if fm_event.get('time_created'):
event_params['time_created'] = timestamp_to_datetime(fm_event['time_created'])
if fm_event.get('time_done'):
event_params['time_done'] = timestamp_to_datetime(fm_event['time_created'])
event = FreshmakerEvent.create_or_update(event_params)[0]
log.debug('Creating Advisory {0}'.format(fm_event['search_key']))
advisory = Advisory.get_or_create(dict(
id_=fm_event['search_key']
))[0]
event.conditional_connect(event.triggered_by_advisory, advisory)
for build_dict in fm_event['builds']:
# To handle a faulty container build in Freshmaker
if build_dict['build_id'] and int(build_dict['build_id']) < 0:
continue
log.debug('Creating FreshmakerBuild {0}'.format(build_dict['build_id']))
fb_params = dict(
id_=build_dict['id'],
dep_on=build_dict['dep_on'],
name=build_dict['name'],
original_nvr=build_dict['original_nvr'],
rebuilt_nvr=build_dict['rebuilt_nvr'],
state=build_dict['state'],
state_name=build_dict['state_name'],
state_reason=build_dict['state_reason'],
time_submitted=timestamp_to_datetime(build_dict['time_submitted']),
type_=build_dict['type'],
type_name=build_dict['type_name'],
url=build_dict['url']
)
if build_dict['time_completed']:
fb_params['time_completed'] = timestamp_to_datetime(
build_dict['time_completed'])
if build_dict['build_id']:
fb_params['build_id'] = build_dict['build_id']
fb = FreshmakerBuild.create_or_update(fb_params)[0]
event.requested_builds.connect(fb)
# The build ID obtained from Freshmaker API is actually a Koji task ID
task_result = None
if build_dict['build_id']:
task_result = self.get_koji_task_result(build_dict['build_id'])
if not task_result:
continue
# Extract the build ID from a task result
xml_root = ET.fromstring(task_result)
# TODO: Change this if a task can trigger multiple builds
try:
build_id = xml_root.find(".//*[name='koji_builds'].//string").text
except AttributeError:
build_id = None
if not build_id:
continue
log.debug('Creating ContainerKojiBuild {0}'.format(build_id))
build_params = {
'id_': build_id,
'original_nvr': build_dict['original_nvr']
}
try:
build = ContainerKojiBuild.create_or_update(build_params)[0]
except neomodel.exceptions.ConstraintValidationFailed:
# This must have errantly been created as a KojiBuild instead of a
# ContainerKojiBuild, so let's fix that.
build = KojiBuild.nodes.get_or_none(id_=build_id)
if not build:
# If there was a constraint validation failure and the build isn't just
# the wrong label, then we can't recover.
raise
build.add_label(ContainerKojiBuild.__label__)
build = ContainerKojiBuild.create_or_update(build_params)[0]
event.successful_koji_builds.connect(build)
if rv_json['meta'].get('next'):
fm_url = rv_json['meta']['next']
else:
break
def query_api_and_update_neo4j(self):
"""
Scrape the Freshmaker API and upload the data to Neo4j.
:param str start_date: a datetime to start scraping data from
"""
# Initialize session and url
session = retry_session()
fm_url = self.freshmaker_url
while True:
log.debug('Querying {0}'.format(fm_url))
rv_json = session.get(fm_url, timeout=15).json()
for fm_event in rv_json['items']:
try:
int(fm_event['search_key'])
except ValueError:
# Skip Freshmaker Events that don't have the search_key as the Advisory ID
continue
event = FreshmakerEvent.create_or_update(dict(
id_=fm_event['id'],
event_type_id=fm_event['event_type_id'],
message_id=fm_event['message_id'],
state=fm_event['state'],
state_name=fm_event['state_name'],
state_reason=fm_event['state_reason'],
url=fm_event['url']
))[0]
advisory = Advisory.get_or_create(dict(
id_=fm_event['search_key']
))[0]
event.conditional_connect(event.triggered_by_advisory, advisory)
for build_dict in fm_event['builds']:
# To handle a faulty container build in Freshmaker
if not build_dict['build_id'] or int(build_dict['build_id']) < 0:
continue
# The build ID obtained from Freshmaker API is actually a Koji task ID
task_result = self.get_koji_task_result(build_dict['build_id'])
if not task_result:
continue
# Extract the build ID from a task result
xml_root = ET.fromstring(task_result)
# TODO: Change this if a task can trigger multiple builds
try:
build_id = xml_root.find(".//*[name='koji_builds'].//string").text
except AttributeError:
build_id = None
if build_id:
build = ContainerKojiBuild.get_or_create(dict(
id_=build_id,
original_nvr=build_dict['original_nvr']
))[0]
event.triggered_container_builds.connect(build)
if rv_json['meta'].get('next'):
fm_url = rv_json['meta']['next']
else:
break