class DependentPackage(ModelMixin):
"""
An identifiable dependent package package object.
"""
purl = String(
repr=True,
label='Dependent package URL',
help='A compact purl package URL. Typically when there is an '
'unresolved requirement, there is no version. '
'If the dependency is resolved, the version should be added to '
'the purl')
extracted_requirement = String(
repr=True,
label='extracted version requirement',
help='String for the original version requirements and constraints. '
'Package-type specific and as found originally in a datafile.')
# ToDo: add `vers` support. See https://github.com/nexB/univers/blob/main/src/univers/version_range.py
scope = String(
repr=True,
label='dependency scope',
help='The scope of this dependency, such as runtime, install, etc. '
'This is package-type specific and is the original scope string.')
is_runtime = Boolean(
default=True,
label='is runtime flag',
help='True if this dependency is a runtime dependency.')
is_optional = Boolean(
default=False,
label='is optional flag',
help='True if this dependency is an optional dependency')
is_resolved = Boolean(
default=False,
label='is resolved flag',
help='True if this dependency version requirement has '
'been resolved and this dependency url points to an '
'exact version.')
resolved_package = Mapping(
label='resolved package data',
help='A mapping of resolved package data for this dependent package, '
'either from the datafile or collected from another source. Some '
'lockfiles for Composer or Cargo contain extra dependency data.')
class GeneratedCodeDetector(ScanPlugin):
"""
Tag a file as generated.
"""
resource_attributes = dict(is_generated=Boolean(
help='True if this file is likely an automatically generated file.'))
sort_order = 50
options = [
CommandLineOption(
('--generated', ),
is_flag=True,
default=False,
help='Classify automatically generated code files with a flag.',
help_group=OTHER_SCAN_GROUP,
sort_order=50,
)
]
def is_enabled(self, generated, **kwargs):
return generated
def get_scanner(self, **kwargs):
return generated_scanner
class TypeDefinition(object):
name = String(repr=True)
filetypes = List(repr=True)
mimetypes = List(repr=True)
extensions = List(repr=True)
strict = Boolean(repr=True,
help=' if True, all criteria must be matched to select this detector.')
class DependentPackage(BaseModel):
"""
An identifiable dependent package package object.
"""
purl = String(
repr=True,
label='Dependent package URL',
help=
'A compact purl package URL. Typically when there is an unresolved requirement, there is no version. '
'If the dependency is resolved, the version should be added to the purl'
)
requirement = String(
repr=True,
label='dependent package version requirement',
help='A string defining version(s)requirements. Package-type specific.'
)
scope = String(
repr=True,
label='dependency scope',
help='The scope of this dependency, such as runtime, install, etc. '
'This is package-type specific and is the original scope string.')
is_runtime = Boolean(
default=True,
label='is runtime flag',
help='True if this dependency is a runtime dependency.')
is_optional = Boolean(
default=False,
label='is optional flag',
help='True if this dependency is an optional dependency')
is_resolved = Boolean(
default=False,
label='is resolved flag',
help='True if this dependency version requirement has '
'been resolved and this dependency url points to an '
'exact version.')
class Gem(object):
"""
A Gem can be packaged as a .gem archive, or it can be a source gem either
fetched from GIT or SVN or from a local path.
"""
supported_opts = 'remote', 'ref', 'revision', 'branch', 'submodules', 'tag',
name = String()
version = String()
platform = String(help='Gem platform')
remote = String(
help=
'remote can be a path, git, svn or Gem repo url. One of GEM, PATH, GIT or SVN'
)
type = String(
# validator=choices(GEM_TYPES),
help='the type of this Gem: One of: {}'.format(', '.join(GEM_TYPES)))
pinned = Boolean()
spec_version = String()
# relative path
path = String()
revision = String(
help='A version control full revision (e.g. a Git commit hash).')
ref = String(
help=
'A version control ref (such as a tag, a shortened revision hash, etc.).'
)
branch = String()
submodules = String()
tag = String()
requirements = List(item_type=String,
help='list of constraints such as ">= 1.1.9"')
dependencies = Mapping(
help='a map of direct dependent Gems, keyed by name',
value_type='Gem',
)
def refine(self):
"""
Apply some refinements to the Gem based on its type:
- fix version and revisions for Gems checked-out from VCS
"""
if self.type == PATH:
self.path = self.remote
if self.type in (
GIT,
SVN,
):
# FIXME: this likely WRONG
# TODO: this may not be correct for SVN BUT SVN has been abandoned
self.spec_version = self.version
if self.revision and not self.ref:
self.version = self.revision
elif self.revision and self.ref:
self.version = self.revision
elif not self.revision and self.ref:
self.version = self.ref
elif not self.revision and self.ref:
self.version = self.ref
def as_nv_tree(self):
"""
Return a tree of name/versions dependency tuples from self as nested
dicts. The tree root is self. Each key is a name/version tuple.
Values are dicts.
"""
tree = {}
root = (
self.name,
self.version,
)
tree[root] = {}
for _name, gem in self.dependencies.items():
tree[root].update(gem.as_nv_tree())
return tree
def flatten(self):
"""
Return a sorted flattened list of unique parent/child tuples.
"""
flattened = []
seen = set()
for gem in self.dependencies.values():
snv = self.type, self.name, self.version
gnv = gem.type, gem.name, gem.version
rel = self, gem
rel_key = snv, gnv
if rel_key not in seen:
flattened.append(rel)
seen.add(rel_key)
for rel in gem.flatten():
parent, child = rel
pnv = parent.type, parent.name, parent.version
cnv = child.type, child.name, child.version
rel_key = pnv, cnv
if rel_key not in seen:
flattened.append(rel)
seen.add(rel_key)
return sorted(flattened)
def dependency_tree(self):
"""
Return a tree of dependencies as nested mappings.
Each key is a "name@version" string and values are dicts.
"""
tree = {}
root = '{}@{}'.format(self.name or '', self.version or '')
tree[root] = {}
for _name, gem in self.dependencies.items():
tree[root].update(gem.dependency_tree())
return tree
def to_dict(self):
"""
Return a native mapping for this Gem.
"""
return dict([
('name', self.name),
('version', self.version),
('platform', self.platform),
('pinned', self.pinned),
('remote', self.remote),
('type', self.type),
('path', self.path),
('revision', self.revision),
('ref', self.ref),
('branch', self.branch),
('submodules', self.submodules),
('tag', self.tag),
('requirements', self.requirements),
('dependencies', self.dependency_tree()),
])
@property
def gem_name(self):
return '{}-{}.gem'.format(self.name, self.version)
class FileClassifier(PreScanPlugin):
"""
Classify a file such as a COPYING file or a package manifest with a flag.
"""
resource_attributes = OrderedDict([
('is_legal',
Boolean(help='True if this file is likely a "legal", license-related'
'file such as a COPYING or LICENSE file.')),
('is_manifest',
Boolean(help='True if this file is likely a package manifest file such '
'as a Maven pom.xml or an npm package.json')),
('is_readme',
Boolean(help='True if this file is likely a README file.')),
('is_top_level',
Boolean(help='True if this file is "top-level" file located either at '
'the root of a package or in a well-known common location.')),
('is_key_file',
Boolean(help='True if this file is "top-level" file and either a '
'legal, readme or manifest file.')),
# ('is_doc',
# Boolean(help='True if this file is likely a documentation file.')),
#
# ('is_test',
# Boolean(help='True if this file is likely a test file.')),
#
# ('is_generated',
# Boolean(help='True if this file is likely an automatically generated file.')),
#
# ('is_build',
# Boolean(help='True if this file is likely a build script or file such as Makefile.')),
#
# we have an is_data attribute
# ('is_data',
# Boolean(help='True if this file is likely data file.')),
])
sort_order = 50
options = [
CommandLineOption(('--classify',),
is_flag=True, default=False,
help='Classify files with flags telling if the file is a legal, '
'or readme or test file, etc.',
help_group=PRE_SCAN_GROUP,
sort_order=50,
)
]
def is_enabled(self, classify, **kwargs):
return classify
def process_codebase(self, codebase, classify, **kwargs):
if not classify:
return
# find the real root directory
real_root = codebase.lowest_common_parent()
if not real_root:
real_root = codebase.root
real_root_dist = real_root.distance(codebase)
for resource in codebase.walk(topdown=True):
real_dist = resource.distance(codebase) - real_root_dist
# this means this is either a child of the root dir or the root itself.
resource.is_top_level = (real_dist < 2)
if resource.is_file:
# TODO: should we do something about directories? for now we only consider files
set_classification_flags(resource)
resource.save(codebase)
