def test_set_component_names_merge_multiple_components():
"""Test that set_component_names merges all the matching components."""
c1 = Component.get_or_create_singleton('', 'python-requests', 'pypi')
c1.alternative_names = ['py-requests', 'requests']
c1.save()
sl1 = SourceLocation(url='http://domain.local/python-requests', type_='local').save()
c1.source_locations.connect(sl1)
c2 = Component.get_or_create_singleton('', 'python2-requests', 'pypi')
c2.alternative_names = ['python3-requests', 'requests']
c2.save()
sl2 = SourceLocation(url='http://domain.local/python2-requests', type_='local').save()
c2.source_locations.connect(sl2)
sl3 = SourceLocation(url='http://domain.local/requests', type_='local').save()
c2.source_locations.connect(sl3)
modify.set_component_names('requests', 'pypi', alternatives=['python-requests'])
component = Component.nodes.get_or_none(
canonical_namespace='', canonical_name='requests', canonical_type='pypi')
assert component
assert component.canonical_name == 'requests'
assert set(component.alternative_names) == set([
'py-requests', 'python2-requests', 'python3-requests', 'python-requests'])
assert len(component.source_locations) == 3
assert Component.nodes.get_or_none(canonical_name='python-requests') is None
assert Component.nodes.get_or_none(canonical_name='python2-requests') is None
def test_component_invalid_create_or_update():
"""Ensure that the Component create_or_update method is not avialable."""
with pytest.raises(RuntimeError):
Component.create_or_update({
'canonical_namespace': 'a',
'canonical_name': 'kernel',
'canonical_type': 'rpm'
})
def test_set_component_names_add_alt_names():
"""Test that set_component_names can add alternative names to an existing component."""
Component.get_or_create_singleton('', 'requests', 'pypi')
modify.set_component_names('requests', 'pypi', alternatives=['python-requests'])
component = Component.nodes.get_or_none(
canonical_namespace='', canonical_name='requests', canonical_type='pypi')
assert component
assert component.alternative_names == ['python-requests']
def test_supersedes_order_container():
"""Test that conatiner SourceLocations are associated in the correct order."""
rpm_comp = Component.get_or_create_singleton('a', 'test', 'docker')
URL_BASE = 'www.example.com/some/package#'
analyzer = main_analyzer.MainAnalyzer()
def get_sl(version):
return analyzer.create_or_update_source_location(
url=URL_BASE + str(random.randint(0, 10000)),
component=rpm_comp,
canonical_version=version)
# named for what order they should come in, created in a random order
fourth = get_sl('v4.0.0-0.27.0.0')
fifth = get_sl('v4.0.0-0.27.0.1')
third = get_sl('v3.11.45-7')
first = get_sl('v3.10.16-4')
second = get_sl('v3.10.45-7')
assert first.previous_version.single() is None
assert second == first.next_version.single()
assert third == second.next_version.single()
assert fifth == fourth.next_version.single()
assert fifth.next_version.single() is None
def test_supersedes_order_maven():
"""Test that Maven SourceLocations are associated in the correct order."""
rpm_comp = Component.get_or_create_singleton('a', 'test', 'java')
URL_BASE = 'www.example.com/some/package#'
analyzer = main_analyzer.MainAnalyzer()
def get_sl(version):
return analyzer.create_or_update_source_location(
url=URL_BASE + str(random.randint(0, 10000)),
component=rpm_comp,
canonical_version=version)
# named for what order they should come in, created in a random order
fourth = get_sl('7.1.0.fuse-710017-redhat-00002')
fifth = get_sl('7.1.0.fuse-710018-redhat-00001')
third = get_sl('7.1.0.fuse-710017-redhat-00001')
first = get_sl('6.2.1.redhat-222')
second = get_sl('6.3.0.redhat-356')
assert first.previous_version.single() is None
assert second == first.next_version.single()
assert third == second.next_version.single()
assert fifth == fourth.next_version.single()
assert fifth.next_version.single() is None
def test_supersedes_order_rpm():
"""Test that RPM SourceLocations are associated in the correct order."""
rpm_comp = Component.get_or_create_singleton('another', 'test', 'rpm')
URL_BASE = 'www.example.com/some/package#'
analyzer = main_analyzer.MainAnalyzer()
def get_sl(version):
return analyzer.create_or_update_source_location(
url=URL_BASE + str(random.randint(0, 10000)),
component=rpm_comp,
canonical_version=version)
# named for what order they should come in, created in a random order
fourth = get_sl('1-1.1-1.el7')
fifth = get_sl('1-1.1a-1.el7')
third = get_sl('0-3-1.el7')
first = get_sl('0-2-2.el7')
second = get_sl('0-2.1-1.el7')
assert first.previous_version.single() is None
assert second == first.next_version.single()
assert third == second.next_version.single()
assert fifth == fourth.next_version.single()
assert fifth.next_version.single() is None
def test_component_get_or_create_singleton():
"""Ensure that the Component.get_or_create_singleton method works correctly."""
c1 = Component.get_or_create_singleton('', 'requests', 'rpm')
assert c1.id # Is saved.
assert c1.canonical_namespace == ''
assert c1.canonical_name == 'requests'
assert c1.canonical_type == 'rpm'
c1.alternative_names = ['python2-requests', 'python2-requests']
c1.save()
# Same name as before
c2 = Component.get_or_create_singleton('', 'requests', 'rpm')
assert c1 == c2
# Different name, but in the alternatives list
c3 = Component.get_or_create_singleton('', 'python2-requests', 'rpm')
assert c1 == c3
def test_set_component_names_fix_canonical_name_no_alt_input():
"""Test that set_component_names fixes a canonical name with alternatives input."""
c = Component.get_or_create_singleton('', 'python-requests', 'pypi')
c.alternative_names = ['requests']
c.save()
modify.set_component_names('requests', 'pypi')
component = Component.nodes.get_or_none(
canonical_namespace='', canonical_name='requests', canonical_type='pypi')
assert component
assert component.canonical_name == 'requests'
assert component.alternative_names == ['python-requests']
def build(cls, name=None, **values):
"""Create an instance of a Component model.
:param name: specific component from the COMPONENTS dict
:param values: specific attributes and their values that override the defaults
:return: model instance with filled-in data
:rtype: assayist.common.models.source.Component
"""
component = cls.COMPONENTS[name] if name else choice(
tuple(cls.COMPONENTS.keys()))
component_attrs = ('canonical_namespace', 'canonical_name',
'canonical_type', 'alternative_names')
data = dict(zip(component_attrs, component))
data.update(values)
return Component(**data)
def test_get_current_and_previous_versions():
"""Test the get_current_and_previous_versions function."""
go = Component(canonical_name='golang',
canonical_type='generic',
canonical_namespace='redhat').save()
next_sl = None
url = 'git://pkgs.domain.local/rpms/golang?#fed96461b05c0078e537c93a3fe974e8b334{version}'
for version in ('1.9.7', '1.9.6', '1.9.5', '1.9.4', '1.9.3'):
sl = SourceLocation(url=url.format(version=version.replace('.', '')),
canonical_version=version,
type_='local').save()
sl.component.connect(go)
if next_sl:
next_sl.previous_version.connect(sl)
next_sl = sl
rv = query.get_current_and_previous_versions('golang', 'generic', '1.9.6')
versions = set([result['canonical_version'] for result in rv])
assert versions == set(['1.9.6', '1.9.5', '1.9.4', '1.9.3'])
def test_create_or_update_source_location_bad(m_conditional_connect):
"""Test that create_or_update_source_location rolls back on error."""
m_conditional_connect.side_effect = ValueError('something broke')
rpm_comp = Component.get_or_create_singleton('a', 'test', 'rpm')
analyzer = main_analyzer.MainAnalyzer()
url = 'www.whatever.com'
canonical_version = '0-1-3'
raised = False
try:
analyzer.create_or_update_source_location(
url=url, component=rpm_comp, canonical_version=canonical_version)
except ValueError:
raised = True
assert raised
sl = SourceLocation.nodes.get_or_none(url=url, type_='upstream')
assert not sl
def test_create_or_update_source_location():
"""Test the basic function of the create_or_update_source_location function."""
rpm_comp = Component.get_or_create_singleton('a', 'test', 'rpm')
analyzer = main_analyzer.MainAnalyzer()
url = 'www.whatever.com'
canonical_version = '0-1-2'
sl = analyzer.create_or_update_source_location(
url=url, component=rpm_comp, canonical_version=canonical_version)
assert sl.url == url
assert sl.canonical_version == sl.canonical_version
assert sl.type_ == 'upstream'
assert rpm_comp in sl.component
sl.id # exists, hence is saved
# 're-creating' should just return existing node
sl2 = analyzer.create_or_update_source_location(
url=url, component=rpm_comp, canonical_version=canonical_version)
assert sl.id == sl2.id
def set_component_names(c_name, c_type, c_namespace='', alternatives=None):
"""
Create, update, and/or merge components to ensure one canonical component remains.
:param str c_name: the canonical name of the component
:param str c_type: the canonical type of the component
:param str c_namespace: the canonical namespace of the component
:kwarg list alternatives: the alternative names associated with the component
:raises ValueError: if the input isn't the proper type or c_name's value is also in alternatives
"""
for arg in (c_name, c_type, c_namespace):
if not isinstance(arg, str):
raise ValueError('c_name, c_type, and c_namespace must be strings')
for arg in (c_name, c_type):
if arg == '':
raise ValueError('c_name and c_type cannot be empty')
if alternatives is None:
alternatives = []
if type(alternatives) not in (list, tuple, set):
raise ValueError(
'The alternatives keyword argument must be None or a list, tuple, or set'
)
for alternative in alternatives:
if not isinstance(alternative, str):
raise ValueError(
'The alternatives keyword argument must only contain strings')
# Create a WHERE clause that checks to see if there is a component with the canonical name or
# alternative name with the passed in canonical name and alternatives
names_where_clause = [
'c.canonical_name = "{0}" OR "{0}" in c.alternative_names'.format(name)
for name in itertools.chain([c_name], alternatives)
]
query = """
MATCH (c:Component {{canonical_type: "{}", canonical_namespace: "{}" }})
WHERE {}
RETURN c
""".format(c_type, c_namespace, ' OR '.join(names_where_clause))
results, _ = neomodel.db.cypher_query(query)
components = [Component.inflate(row[0]) for row in results]
# If no matching component is returned, just create one
if not components:
component = Component(canonical_namespace=c_namespace,
canonical_name=c_name,
canonical_type=c_type,
alternative_names=alternatives).save()
log.info(f'Creating the component "{component}"')
return
# Merge all the current canonical names and alternative names
all_alt_names = set(alternatives)
for c in components:
all_alt_names.add(c.canonical_name)
all_alt_names.update(c.alternative_names)
# By removing the correct canonical name, we have a set of the correct alternative names
all_alt_names.discard(c_name)
# This will be the only remaining component if there is more than one component returned, as
# the information stored in the others will be merged into this one
component = components[0]
if component.canonical_name != c_name:
log.info(f'Setting the canonical name of "{c_name}" on "{component}"')
component.canonical_name = c_name
if set(component.alternative_names) != set(all_alt_names):
log.info(
f'Setting the alternative names of "{all_alt_names}" on "{component}"'
)
component.alternative_names = list(all_alt_names)
# If there was more than one component that matched, we must merge them
for c in components[1:]:
log.info(
f'Merging the source locations of "{c}" in the existing component "{component}"'
)
# Merge all the source locations on the first component
for sl in c.source_locations.all():
component.source_locations.connect(sl)
# Delete this component since all its information is now stored in the first component
log.warning(
f'Deleting the component "{c}" since it was merged into "{component}"'
)
c.delete()
component.save()
def get_source_components_for_build(build_id):
"""
Get source code components used in a build.
In more detail:
Get the canonical names of
the sources ultimately used for
artifacts ultimately embedded in
the artifact from a build.
For example: for a container image build, get the names of:
* Upstream projects for RPMs installed in the images
* Go packages used to build executables in the images
etc
The canonical names include versions where available.
This function is implemented using two DB queries. The first
establishes the artifacts our query relates to, as well any
relationships between them. The second finds the information
needed to establish source code versions relating to those
artifacts.
:param int build_id: the Koji build's ID
:return: a tree of artifacts and the source code components
:rtype: dict
"""
build = Build.nodes.get_or_none(id_=str(build_id))
if not build:
raise NotFound('The requested build does not exist in the database')
query = """
// Find artifacts which (or artifacts which embed artifacts which)...
//
// (Note: "*0.." means zero or more edges; if zero edges, 'a' is the
// artifact directly produced by the build.)
MATCH (a:Artifact) <-[e:EMBEDS*0..]- (:Artifact)
// Were produced by the build
<-[:PRODUCED]- (:Build {{id: {0} }})
// Return the artifacts and relationships
RETURN a, e
""".format(repr(str(build_id)))
results, _ = neomodel.db.cypher_query(query)
artifact_dbids = set()
artifacts_by_id = {}
for artifact in [Artifact.inflate(a) for a, _ in results]:
artifacts_by_id[(artifact.type_, artifact.archive_id)] = {
'artifact': {
key: getattr(artifact, key)
for key in ('architecture', 'filename')
}
}
artifact_dbids.add(artifact.id)
embedded_artifacts = set() # needed when we build a tree later
for _, edges in results:
for edge in edges:
embeds, embedded = [Artifact.inflate(node) for node in edge.nodes]
by_id = artifacts_by_id[(embeds.type_, embeds.archive_id)]
embeds_list = by_id.setdefault('embeds_ids', set())
index = (embedded.type_, embedded.archive_id)
embeds_list.add(index)
embedded_artifacts.add(index)
query = """
// Find the artifacts
MATCH (a:Artifact) WHERE id(a) IN [{0}]
// Find the builds that produced all of those
// (this includes the original build)
MATCH (a) <-[:PRODUCED]- (:Build)
// Find the source each was built from
-[:BUILT_FROM]-> (:SourceLocation)
// Include upstream or vendored source locations
//
// Note: this is 0 or more relationships, each of which
// may be either UPSTREAM or EMBEDS
-[:UPSTREAM|EMBEDS*0..]-> (sl:SourceLocation)
// Find the components these locations are source for
-[:SOURCE_FOR]-> (c:Component)
// Only include source locations with no further upstream
WHERE NOT (sl) -[:UPSTREAM]-> (:SourceLocation)
RETURN a, sl, c
""".format(','.join(repr(dbid) for dbid in artifact_dbids))
results, _ = neomodel.db.cypher_query(query)
for a, sl, c in results:
artifact = Artifact.inflate(a)
sourceloc = SourceLocation.inflate(sl)
component = Component.inflate(c)
metadata = artifacts_by_id[(artifact.type_, artifact.archive_id)]
sources = metadata.setdefault('sources', [])
pieces = {}
for piece in ('type', 'namespace', 'name'):
pieces[piece] = getattr(component, 'canonical_{}'.format(piece))
pieces['version'] = sourceloc.canonical_version
pieces['qualifiers'] = {'download_url': sourceloc.url}
sources.append(pieces)
# Build a tree of artifacts
artifacts = {}
toplevel = [
key for key, value in artifacts_by_id.items()
if key not in embedded_artifacts
]
def construct(aid):
a = artifacts_by_id[aid]
try:
embeds_ids = a.pop('embeds_ids')
except KeyError:
return a
a['embeds'] = {}
for embedded_id in embeds_ids:
a['embeds'][embedded_id] = construct(embedded_id)
return a
for aid in toplevel:
artifacts[aid] = construct(aid)
return artifacts