def event_state_handler(self, msg): """ Handle a Freshmaker event state changed message and update Neo4j if necessary. :param dict msg: a message to be processed """ msg_id = msg['body']['msg']['message_id'] if msg['body']['msg'].get('dry_run'): return event_params = { 'id_': str(msg['body']['msg']['id']), 'state_name': msg['body']['msg']['state_name'], 'state_reason': msg['body']['msg']['state_reason'] } if 'time_created' in msg['body']['msg']: event_params['time_created'] = timestamp_to_datetime( msg['body']['msg']['time_created']) if 'time_done' in msg['body']['msg'] and msg['body']['msg'][ 'time_done'] is not None: event_params['time_done'] = timestamp_to_datetime( msg['body']['msg']['time_done']) event = FreshmakerEvent.create_or_update(event_params)[0] advisory_name = msg_id.rsplit('.', 1)[-1] if advisory_name[0:4] not in ('RHSA', 'RHBA', 'RHEA'): log.warn( 'Unable to parse the advisory name from the Freshmaker message_id: {0}' .format(msg_id)) advisory_name = None advisory = Advisory.get_or_create({ 'id_': msg['body']['msg']['search_key'], 'advisory_name': advisory_name })[0] event.conditional_connect(event.triggered_by_advisory, advisory)
def event_state_handler(self, msg): """ Handle a Freshmaker event state changed message and update Neo4j if necessary. :param dict msg: a message to be processed """ msg_id = msg['body']['msg']['message_id'] event = FreshmakerEvent.create_or_update({ 'id_': str(msg['body']['msg']['id']), 'event_type_id': msg['body']['msg']['event_type_id'], 'message_id': msg_id, 'state': msg['body']['msg']['state'], 'state_name': msg['body']['msg']['state_name'], 'state_reason': msg['body']['msg']['state_reason'] })[0] advisory_name = msg_id.rsplit('.', 1)[-1] if advisory_name[0:4] not in ('RHSA', 'RHBA', 'RHEA'): log.warn( 'Unable to parse the advisory name from the Freshmaker message_id: {0}' .format(msg_id)) advisory_name = None advisory = Advisory.get_or_create({ 'id_': msg['body']['msg']['search_key'], 'advisory_name': advisory_name })[0] event.conditional_connect(event.triggered_by_advisory, advisory)
def query_api_and_update_neo4j(self): """ Scrape the Freshmaker API and upload the data to Neo4j. :param str start_date: a datetime to start scraping data from """ # Initialize session and url session = retry_session() fm_url = self.freshmaker_url while True: log.debug('Querying {0}'.format(fm_url)) try: rv_json = session.get(fm_url, timeout=60).json() except ConnectionError: # TODO: Remove this once FACTORY-3955 is resolved log.error( 'The connection to Freshmaker at %s failed. Skipping the rest of the scraper.', fm_url, ) break for fm_event in rv_json['items']: try: int(fm_event['search_key']) except ValueError: # Skip Freshmaker Events that don't have the search_key as the Advisory ID continue log.debug('Creating FreshmakerEvent {0}'.format(fm_event['id'])) event_params = dict( id_=fm_event['id'], event_type_id=fm_event['event_type_id'], message_id=fm_event['message_id'], state=fm_event['state'], state_name=fm_event['state_name'], state_reason=fm_event['state_reason'], url=fm_event['url'] ) if fm_event.get('time_created'): event_params['time_created'] = timestamp_to_datetime(fm_event['time_created']) if fm_event.get('time_done'): event_params['time_done'] = timestamp_to_datetime(fm_event['time_created']) event = FreshmakerEvent.create_or_update(event_params)[0] log.debug('Creating Advisory {0}'.format(fm_event['search_key'])) advisory = Advisory.get_or_create(dict( id_=fm_event['search_key'] ))[0] event.conditional_connect(event.triggered_by_advisory, advisory) for build_dict in fm_event['builds']: # To handle a faulty container build in Freshmaker if build_dict['build_id'] and int(build_dict['build_id']) < 0: continue log.debug('Creating FreshmakerBuild {0}'.format(build_dict['build_id'])) fb_params = dict( id_=build_dict['id'], dep_on=build_dict['dep_on'], name=build_dict['name'], original_nvr=build_dict['original_nvr'], rebuilt_nvr=build_dict['rebuilt_nvr'], state=build_dict['state'], state_name=build_dict['state_name'], state_reason=build_dict['state_reason'], time_submitted=timestamp_to_datetime(build_dict['time_submitted']), type_=build_dict['type'], type_name=build_dict['type_name'], url=build_dict['url'] ) if build_dict['time_completed']: fb_params['time_completed'] = timestamp_to_datetime( build_dict['time_completed']) if build_dict['build_id']: fb_params['build_id'] = build_dict['build_id'] fb = FreshmakerBuild.create_or_update(fb_params)[0] event.requested_builds.connect(fb) # The build ID obtained from Freshmaker API is actually a Koji task ID task_result = None if build_dict['build_id']: task_result = self.get_koji_task_result(build_dict['build_id']) if not task_result: continue # Extract the build ID from a task result xml_root = ET.fromstring(task_result) # TODO: Change this if a task can trigger multiple builds try: build_id = xml_root.find(".//*[name='koji_builds'].//string").text except AttributeError: build_id = None if not build_id: continue log.debug('Creating ContainerKojiBuild {0}'.format(build_id)) build_params = { 'id_': build_id, 'original_nvr': build_dict['original_nvr'] } try: build = ContainerKojiBuild.create_or_update(build_params)[0] except neomodel.exceptions.ConstraintValidationFailed: # This must have errantly been created as a KojiBuild instead of a # ContainerKojiBuild, so let's fix that. build = KojiBuild.nodes.get_or_none(id_=build_id) if not build: # If there was a constraint validation failure and the build isn't just # the wrong label, then we can't recover. raise build.add_label(ContainerKojiBuild.__label__) build = ContainerKojiBuild.create_or_update(build_params)[0] event.successful_koji_builds.connect(build) if rv_json['meta'].get('next'): fm_url = rv_json['meta']['next'] else: break
def query_api_and_update_neo4j(self): """ Scrape the Freshmaker API and upload the data to Neo4j. :param str start_date: a datetime to start scraping data from """ # Initialize session and url session = retry_session() fm_url = self.freshmaker_url while True: log.debug('Querying {0}'.format(fm_url)) rv_json = session.get(fm_url, timeout=15).json() for fm_event in rv_json['items']: try: int(fm_event['search_key']) except ValueError: # Skip Freshmaker Events that don't have the search_key as the Advisory ID continue event = FreshmakerEvent.create_or_update(dict( id_=fm_event['id'], event_type_id=fm_event['event_type_id'], message_id=fm_event['message_id'], state=fm_event['state'], state_name=fm_event['state_name'], state_reason=fm_event['state_reason'], url=fm_event['url'] ))[0] advisory = Advisory.get_or_create(dict( id_=fm_event['search_key'] ))[0] event.conditional_connect(event.triggered_by_advisory, advisory) for build_dict in fm_event['builds']: # To handle a faulty container build in Freshmaker if not build_dict['build_id'] or int(build_dict['build_id']) < 0: continue # The build ID obtained from Freshmaker API is actually a Koji task ID task_result = self.get_koji_task_result(build_dict['build_id']) if not task_result: continue # Extract the build ID from a task result xml_root = ET.fromstring(task_result) # TODO: Change this if a task can trigger multiple builds try: build_id = xml_root.find(".//*[name='koji_builds'].//string").text except AttributeError: build_id = None if build_id: build = ContainerKojiBuild.get_or_create(dict( id_=build_id, original_nvr=build_dict['original_nvr'] ))[0] event.triggered_container_builds.connect(build) if rv_json['meta'].get('next'): fm_url = rv_json['meta']['next'] else: break