def update_neo4j(self, advisories):
"""
Update Neo4j with Errata Tool advisories from Teiid.
:param list advisories: a list of dictionaries of advisories
"""
count = 0
for advisory in advisories:
count += 1
log.info('Processing advisory {0}/{1}'.format(
count, len(advisories)))
# The content_types column is a string with YAML in it, so convert it to a list
content_types = yaml.safe_load(advisories[0]['content_types'])
adv = Advisory.create_or_update({
'actual_ship_date':
advisory['actual_ship_date'],
'advisory_name':
advisory['advisory_name'],
'content_types':
content_types,
'created_at':
advisory['created_at'],
'id_':
advisory['id'],
'issue_date':
advisory['issue_date'],
'product_name':
advisory['product_name'],
'product_short_name':
advisory['product_short_name'],
'release_date':
advisory['release_date'],
'security_impact':
advisory['security_impact'],
'security_sla':
advisory['security_sla'],
'state':
advisory['state'],
'status_time':
advisory['status_time'],
'synopsis':
advisory['synopsis'],
'type_':
advisory['type'],
'update_date':
advisory['update_date'],
'updated_at':
advisory['updated_at']
})[0]
assigned_to = User.get_or_create(
{'username': advisory['assigned_to'].split('@')[0]})[0]
adv.conditional_connect(adv.assigned_to, assigned_to)
package_owner = User.get_or_create(
{'username': advisory['package_owner'].split('@')[0]})[0]
adv.conditional_connect(adv.package_owner, package_owner)
reporter = User.get_or_create(
{'username': advisory['reporter'].split('@')[0]})[0]
adv.conditional_connect(adv.reporter, reporter)
for state in self.get_advisory_states(advisory['id']):
adv_state = AdvisoryState.create_or_update({
'id_':
state['id'],
'name':
state['name'],
'created_at':
state['created_at'],
'updated_at':
state['updated_at']
})[0]
adv_state.conditional_connect(adv_state.advisory, adv)
state_creator = User.get_or_create(
{'username': state['username'].split('@')[0]})[0]
adv_state.conditional_connect(adv_state.creator, state_creator)
for attached_bug in self.get_attached_bugs(advisory['id']):
bug = BugzillaBug.get_or_create(attached_bug)[0]
adv.attached_bugs.connect(bug)
for associated_build in self.get_associated_builds(advisory['id']):
# If this is set, that means it was once part of the advisory but not anymore.
# This relationship needs to be deleted if it exists.
if associated_build['removed_index_id']:
build = KojiBuild.nodes.get_or_none(
id_=associated_build['id_'])
if build:
adv.attached_builds.disconnect(build)
else:
# This key shouldn't be stored in Neo4j
del associated_build['removed_index_id']
build = KojiBuild.get_or_create(associated_build)[0]
adv.attached_builds.connect(build)
def advisory_handler(self, msg):
"""
Handle an Errata tool advisory changes and update Neo4j if necessary.
:param dict msg: a message to be processed
"""
advisory_id = msg['body']['headers']['errata_id']
erratum_url = '{0}/api/v1/erratum/{1}'.format(
self.config['estuary_updater.errata_url'].rstrip('/'), advisory_id)
response = requests.get(erratum_url,
auth=requests_kerberos.HTTPKerberosAuth(),
timeout=10)
advisory_json = response.json()
advisory_type = msg['body']['headers']['type'].lower()
advisory_info = advisory_json['errata'][advisory_type]
embargoed = msg['body']['headers']['synopsis'] == 'REDACTED'
# We can't store information on embargoed advisories other than the ID
if not embargoed:
product_url = '{0}/products/{1}.json'.format(
self.config['estuary_updater.errata_url'].rstrip('/'),
advisory_info['product_id'])
response = requests.get(product_url,
auth=requests_kerberos.HTTPKerberosAuth(),
timeout=10)
product_json = response.json()
reporter_url = '{0}/api/v1/user/{1}'.format(
self.config['estuary_updater.errata_url'].rstrip('/'),
advisory_info['reporter_id'])
response = requests.get(reporter_url,
auth=requests_kerberos.HTTPKerberosAuth(),
timeout=10)
reporter_json = response.json()
reporter = User.create_or_update({
'username':
reporter_json['login_name'].split('@')[0],
'email':
reporter_json['email_address']
})[0]
assigned_to_url = '{0}/api/v1/user/{1}'.format(
self.config['estuary_updater.errata_url'].rstrip('/'),
advisory_info['assigned_to_id'])
response = requests.get(assigned_to_url,
auth=requests_kerberos.HTTPKerberosAuth(),
timeout=10)
assigned_to_json = response.json()
assigned_to = User.create_or_update({
'username':
assigned_to_json['login_name'].split('@')[0],
'email':
assigned_to_json['email_address']
})[0]
advisory_params = {
'advisory_name': advisory_info['fulladvisory'],
'id_': advisory_id,
'product_name': product_json['product']['name'],
'security_impact': advisory_info['security_impact'],
'state': advisory_info['status'],
'synopsis': msg['body']['headers']['synopsis']
}
for dt in ('actual_ship_date', 'created_at', 'issue_date',
'release_date', 'security_sla', 'status_updated_at',
'update_date'):
if advisory_info[dt]:
if dt == 'status_updated_at':
estuary_key = 'status_time'
else:
estuary_key = dt
advisory_params[estuary_key] = timestamp_to_datetime(
advisory_info[dt])
else:
advisory_params = {
'id_': advisory_id,
# Set this to REDACTED and it'll be updated when it becomes public
'advisory_name': 'REDACTED'
}
if 'docker' in advisory_info['content_types']:
try:
advisory = ContainerAdvisory.create_or_update(
advisory_params)[0]
except neomodel.exceptions.ConstraintValidationFailed:
# This must have errantly been created as an Advisory instead of a
# ContainerAdvisory, so let's fix that.
advisory = Advisory.nodes.get_or_none(id_=advisory_id)
if not advisory:
# If there was a constraint validation failure and the advisory isn't just
# the wrong label, then we can't recover.
raise
advisory.add_label(ContainerAdvisory.__label__)
advisory = ContainerAdvisory.create_or_update(
advisory_params)[0]
else:
# Check to see if a ContainerAdvisory using this id already exists, and if so remove its
# label because it should not be a ContainerAdvisory if docker isn't a content type.
container_adv = ContainerAdvisory.nodes.get_or_none(
id_=advisory_id)
if container_adv:
container_adv.remove_label(ContainerAdvisory.__label__)
advisory = Advisory.create_or_update(advisory_params)[0]
if not embargoed:
advisory.conditional_connect(advisory.reporter, reporter)
advisory.conditional_connect(advisory.assigned_to, assigned_to)
bugs = advisory_json['bugs']['bugs']
for bug in bugs:
bug = BugzillaBug.get_or_create({'id_': bug['bug']['id']})[0]
advisory.attached_bugs.connect(bug)
def update_neo4j(self, advisories):
"""
Update Neo4j with Errata Tool advisories from Teiid.
:param list advisories: a list of dictionaries of advisories
"""
count = 0
for advisory in advisories:
count += 1
log.info('Processing advisory {0}/{1}'.format(
count, len(advisories)))
# The content_types column is a string with YAML in it, so convert it to a list
content_types = yaml.safe_load(advisory['content_types'])
adv = Advisory.create_or_update({
'actual_ship_date':
advisory['actual_ship_date'],
'advisory_name':
advisory['advisory_name'],
'content_types':
content_types,
'created_at':
advisory['created_at'],
'id_':
advisory['id'],
'issue_date':
advisory['issue_date'],
'product_name':
advisory['product_name'],
'product_short_name':
advisory['product_short_name'],
'release_date':
advisory['release_date'],
'security_impact':
advisory['security_impact'],
'security_sla':
advisory['security_sla'],
'state':
advisory['state'],
'status_time':
advisory['status_time'],
'synopsis':
advisory['synopsis'],
'update_date':
advisory['update_date'],
})[0]
container_adv = False
for associated_build in self.get_associated_builds(advisory['id']):
# Even if a node has two labels in the database, Neo4j returns the node
# only with the specific label you asked for. Hence we check for labels
# ContainerKojiBuild and KojiBuild separately for the same node.
build = ContainerKojiBuild.nodes.get_or_none(
id_=associated_build['id_'])
if not build:
build = KojiBuild.nodes.get_or_none(
id_=associated_build['id_'])
if build and not container_adv:
if build.__label__ == 'ContainerKojiBuild':
adv.add_label(ContainerAdvisory.__label__)
container_adv = True
# If this is set, that means it was once part of the advisory but not anymore.
# This relationship needs to be deleted if it exists.
if associated_build['removed_index_id']:
if build:
adv.attached_builds.disconnect(build)
else:
# Query Teiid and create the entry only if the build is not present in Neo4j
if not build:
attached_build = self.get_koji_build(
associated_build['id_'])
if attached_build:
if self.is_container_build(attached_build):
build = ContainerKojiBuild.get_or_create(
{'id_': associated_build['id_']})[0]
else:
build = KojiBuild.get_or_create(
{'id_': associated_build['id_']})[0]
# This will happen only if we do not find the build we are looking for in Teiid
# which shouldn't usually happen under normal conditions
if not build:
log.warn(
'The Koji build with ID {} was not found in Teiid!'
.format(associated_build['id_']))
continue
if adv.__label__ != ContainerAdvisory.__label__ \
and build.__label__ == ContainerKojiBuild.__label__:
adv.add_label(ContainerAdvisory.__label__)
attached_rel = adv.attached_builds.relationship(build)
time_attached = associated_build['time_attached']
if attached_rel:
if attached_rel.time_attached != time_attached:
adv.attached_builds.replace(
build, {'time_attached': time_attached})
else:
adv.attached_builds.connect(
build, {'time_attached': time_attached})
assigned_to = User.get_or_create(
{'username': advisory['assigned_to'].split('@')[0]})[0]
adv.conditional_connect(adv.assigned_to, assigned_to)
reporter = User.get_or_create(
{'username': advisory['reporter'].split('@')[0]})[0]
adv.conditional_connect(adv.reporter, reporter)
for attached_bug in self.get_attached_bugs(advisory['id']):
bug = BugzillaBug.get_or_create(attached_bug)[0]
adv.attached_bugs.connect(bug)