def test_shorthandURLs(self):
for url in ShortHand_URLs:
for s in Git_Specs:
if len(s):
# Shorthand URLs support '@' and ' ' as well as '#'
for m in ['#', '@', ' ']:
sv = sourceparse.parseSourceURL(url + m + s)
self.assertEqual(sv.source_type, 'github')
self.assertEqual(sv.spec, s)
else:
sv = sourceparse.parseSourceURL(url)
self.assertEqual(sv.source_type, 'github')
self.assertEqual(sv.spec, s)
def test_shorthandURLs(self):
for url in ShortHand_URLs:
for s in Git_Specs:
if len(s):
# Shorthand URLs support '@' and ' ' as well as '#'
for m in ['#', '@', ' ']:
sv = sourceparse.parseSourceURL(url + m + s)
self.assertEqual(sv.source_type, 'github')
self.assertEqual(sv.spec, s)
else:
sv = sourceparse.parseSourceURL(url)
self.assertEqual(sv.source_type, 'github')
self.assertEqual(sv.spec, s)
def satisfyDep(dspec):
try:
r = provider(
dspec,
available_components,
search_dirs,
modules_path,
update_installed,
self
)
if r and not sourceparse.parseSourceURL(dspec.versionReq()).semanticSpecMatches(r.getVersion()):
shrinkwrap_msg = ''
if dspec.isShrinkwrapped():
shrinkwrap_msg = 'shrinkwrap on '
msg = 'does not meet specification %s required by %s%s' % (
dspec.versionReq(), shrinkwrap_msg, self.getName()
)
logger.debug('%s %s', r.getName(), msg)
r.setError(msg)
return r
except access_common.Unavailable as e:
errors.append(e)
self.dependencies_failed = True
except vcs.VCSError as e:
errors.append(e)
self.dependencies_failed = True
def remoteComponentFor(name, version_required, registry='modules'):
''' Return a RemoteComponent sublclass for the specified component name and
source url (or version specification)
Raises an exception if any arguments are invalid.
'''
try:
vs = sourceparse.parseSourceURL(version_required)
except ValueError as e:
raise access_common.Unavailable(
'%s' % (e)
)
if vs.source_type == 'registry':
if registry not in ('modules', 'targets'):
raise Exception('no known registry namespace "%s"' % registry)
return registry_access.RegistryThing.createFromSource(
vs, name, registry=registry
)
elif vs.source_type == 'github':
return github_access.GithubComponent.createFromSource(vs, name)
elif vs.source_type == 'git':
return git_access.GitComponent.createFromSource(vs, name)
elif vs.source_type == 'hg':
return hg_access.HGComponent.createFromSource(vs, name)
else:
raise Exception('unsupported module source: "%s"' % vs.source_type)
def satisfyDep(dspec):
try:
r = provider(
dspec,
available_components,
search_dirs,
modules_path,
update_installed,
self
)
if r and not sourceparse.parseSourceURL(dspec.versionReq()).semanticSpecMatches(r.getVersion()):
shrinkwrap_msg = ''
if dspec.isShrinkwrapped():
shrinkwrap_msg = 'shrinkwrap on '
msg = 'does not meet specification %s required by %s%s' % (
dspec.versionReq(), shrinkwrap_msg, self.getName()
)
logger.debug('%s %s', r.getName(), msg)
r.setError(msg)
return r
except access_common.Unavailable as e:
errors.append(e)
self.dependencies_failed = True
except vcs.VCSError as e:
errors.append(e)
self.dependencies_failed = True
def checkDependenciesForShrinkwrap(dependency_list):
''' return a list of errors encountered (e.g. dependency missing or
specification not met
'''
# sourceparse, , parse version specifications, internall
from yotta.lib import sourceparse
errors = []
# first gather the available versions of things:
available_versions = {}
for mod in dependency_list.get('modules', []):
available_versions[mod['name']] = mod['version']
# now check that the available versions satisfy all of the specifications
# from other modules:
for mod in dependency_list.get('modules', []):
for spec_info in mod.get('specifications', []):
name = spec_info['name']
spec = spec_info['version']
if spec_info.get('testOnly', False):
# test-only specifications are ignored for shrinkwrap
continue
if not name in available_versions:
errors.append('dependency %s (required by %s) is missing' % (
name, mod['name']
))
else:
available_version = available_versions[name]
parsed_spec = sourceparse.parseSourceURL(spec)
if not parsed_spec.semanticSpecMatches(available_version):
errors.append('%s@%s does not meet specification %s required by %s' % (
name, available_version, parsed_spec.semanticSpec(), mod['name']
))
return errors
def setUp(self):
ensureGitConfig()
vs = sourceparse.parseSourceURL(Test_Repo)
self.remote_component = git_access.GitComponent.createFromSource(vs, Test_Name)
self.assertTrue(self.remote_component)
self.working_copy = self.remote_component.clone()
self.assertTrue(self.working_copy)
def setUp(self):
ensureGitConfig()
vs = sourceparse.parseSourceURL(Test_Repo)
self.remote_component = git_access.GitComponent.createFromSource(vs, Test_Name)
self.assertTrue(self.remote_component)
self.working_copy = self.remote_component.clone()
self.assertTrue(self.working_copy)
def checkDependenciesForShrinkwrap(dependency_list):
''' return a list of errors encountered (e.g. dependency missing or
specification not met
'''
# sourceparse, , parse version specifications, internall
from yotta.lib import sourceparse
errors = []
# first gather the available versions of things:
available_versions = {}
for mod in dependency_list.get('modules', []):
available_versions[mod['name']] = mod['version']
# now check that the available versions satisfy all of the specifications
# from other modules:
for mod in dependency_list.get('modules', []):
for spec_info in mod.get('specifications', []):
name = spec_info['name']
spec = spec_info['version']
if spec_info.get('testOnly', False):
# test-only specifications are ignored for shrinkwrap
continue
if not name in available_versions:
errors.append('dependency %s (required by %s) is missing' %
(name, mod['name']))
else:
available_version = available_versions[name]
parsed_spec = sourceparse.parseSourceURL(spec)
if not parsed_spec.semanticSpecMatches(available_version):
errors.append(
'%s@%s does not meet specification %s required by %s' %
(name, available_version, parsed_spec.semanticSpec(),
mod['name']))
return errors
def satisfyVersionFromSearchPaths(name, version_required, search_paths, update=False, type='module', inherit_shrinkwrap=None):
''' returns a Component/Target for the specified version, if found in the
list of search paths. If `update' is True, then also check for newer
versions of the found component, and update it in-place (unless it was
installed via a symlink).
'''
# Pack, , base class for targets and components, internal
from yotta.lib import pack
v = None
try:
sv = sourceparse.parseSourceURL(version_required)
except ValueError as e:
logging.error(e)
return None
try:
local_version = searchPathsFor(
name,
sv.semanticSpec(),
search_paths,
type,
inherit_shrinkwrap = inherit_shrinkwrap
)
except pack.InvalidDescription as e:
logger.error(e)
return None
logger.debug("%s %s locally" % (('found', 'not found')[not local_version], name))
if local_version:
if update and not local_version.installedLinked():
#logger.debug('attempt to check latest version of %s @%s...' % (name, version_required))
v = latestSuitableVersion(name, version_required, registry=_registryNamespaceForType(type))
if local_version:
local_version.setLatestAvailable(v)
# if we don't need to update, then we're done
if local_version.installedLinked() or not local_version.outdated():
logger.debug("satisfy component from directory: %s" % local_version.path)
# if a component exists (has a valid description file), and either is
# not outdated, or we are not updating
if name != local_version.getName():
raise Exception('Component %s found in incorrectly named directory %s (%s)' % (
local_version.getName(), name, local_version.path
))
return local_version
# otherwise, we need to update the installed component
logger.info('update outdated: %s@%s -> %s' % (
name,
local_version.getVersion(),
v
))
# must rm the old component before continuing
fsutils.rmRf(local_version.path)
return _satisfyVersionByInstallingVersion(
name, version_required, local_version.path, v, type=type, inherit_shrinkwrap=inherit_shrinkwrap
)
return None
def installComponentAsDependency(args, current_component):
logging.debug('install component %s as dependency of %s' % (args.component, current_component))
if not current_component:
logging.debug(str(current_component.getError()))
logging.error('The current directory does not contain a valid module.')
return -1
target, errors = current_component.satisfyTarget(args.target, additional_config=args.config)
if errors:
for error in errors:
logging.error(error)
return 1
modules_dir = current_component.modulesPath()
from yotta.lib import sourceparse
# check if we have both a name and specification
component_name, component_spec = sourceparse.parseModuleNameAndSpec(args.component)
logging.info('%s, %s', component_name, component_spec)
if component_name == current_component.getName():
logging.error('will not install module %s as a dependency of itself', component_name)
return -1
try:
installed = access.satisfyVersion(
component_name,
component_spec,
available = {current_component.getName():current_component},
search_paths = [modules_dir],
working_directory = modules_dir
)
except access_common.AccessException as e:
logging.error(e)
return 1
# We always add the component to the dependencies of the current component
# (if it is not already present), and write that back to disk. Without
# writing to disk the dependency wouldn't be usable.
if installed and not current_component.hasDependency(component_name):
vs = sourceparse.parseSourceURL(component_spec)
if vs.source_type == 'registry':
saved_spec = current_component.saveDependency(installed)
else:
saved_spec = current_component.saveDependency(installed, component_spec)
current_component.writeDescription()
logging.info('dependency %s: %s written to module.json', component_name, saved_spec)
else:
logging.info('dependency %s is already present in module.json', component_name)
# !!! should only install dependencies necessary for the one thing that
# we're installing (but existing components should be made available to
# satisfy dependencies)
components, errors = current_component.satisfyDependenciesRecursive(
target = target,
available_components = [(current_component.getName(), current_component)],
test = {'own':'toplevel', 'all':True, 'none':False}[args.install_test_deps]
)
return checkPrintStatus(errors, components, current_component, target)
def test_hgURLs(self):
for url in HG_URLs:
for s in HG_Specs:
if len(s):
source = url + '#' + s
else:
source = url
sv = sourceparse.parseSourceURL(source)
self.assertEqual(sv.source_type, 'hg')
self.assertEqual(sv.spec, s)
def test_githubURLs(self):
for url in Github_URLs:
for s in Git_Specs:
if len(s):
source = url + '#' + s
else:
source = url
sv = sourceparse.parseSourceURL(source)
self.assertEqual(sv.source_type, 'github')
self.assertEqual(sv.spec, s)
def test_hgURLs(self):
for url in HG_URLs:
for s in HG_Specs:
if len(s):
source = url + '#' + s
else:
source = url
sv = sourceparse.parseSourceURL(source)
self.assertEqual(sv.source_type, 'hg')
self.assertEqual(sv.spec, s)
def test_githubURLs(self):
for url in Github_URLs:
for s in Git_Specs:
if len(s):
source = url + '#' + s
else:
source = url
sv = sourceparse.parseSourceURL(source)
self.assertEqual(sv.source_type, 'github')
self.assertEqual(sv.spec, s)
def satisfyVersion(
name,
version_required,
available,
search_paths,
working_directory,
update_installed=None,
type='module', # or 'target'
inherit_shrinkwrap=None
):
''' returns a Component/Target for the specified version (either to an already
installed copy (from the available list, or from disk), or to a newly
downloaded one), or None if the version could not be satisfied.
update_installed = None / 'Update'
None: prevent any attempt to look for new versions if the
component/target already exists
Update: replace any existing version with the newest available, if
the newest available has a higher version
'''
r = satisfyFromAvailable(name, available, type=type)
if r is not None:
if not sourceparse.parseSourceURL(version_required).semanticSpecMatches(r.getVersion()):
raise access_common.SpecificationNotMet(
"Installed %s %s doesn't match specification %s" % (type, name, version_required)
)
return r
r = satisfyVersionFromSearchPaths(
name,
version_required,
search_paths,
(update_installed == 'Update'),
type = type,
inherit_shrinkwrap = inherit_shrinkwrap
)
if r is not None:
return r
return satisfyVersionByInstalling(
name, version_required, working_directory, type=type, inherit_shrinkwrap = inherit_shrinkwrap
)
def remoteComponentFor(name, version_required, registry="modules"):
""" Return a RemoteComponent sublclass for the specified component name and
source url (or version specification)
Raises an exception if any arguments are invalid.
"""
vs = sourceparse.parseSourceURL(version_required)
if vs.source_type == "registry":
if registry not in ("modules", "targets"):
raise Exception('no known registry namespace "%s"' % registry)
return registry_access.RegistryThing.createFromSource(vs, name, registry=registry)
elif vs.source_type == "github":
return github_access.GithubComponent.createFromSource(vs, name)
elif vs.source_type == "git":
return git_access.GitComponent.createFromSource(vs, name)
elif vs.source_type == "hg":
return hg_access.HGComponent.createFromSource(vs, name)
else:
raise Exception('unsupported module source: "%s"' % vs.source_type)
def test_versionSpec(self):
vs = sourceparse.parseSourceURL(Test_Repo_With_Spec)
spec = git_access.GitComponent.createFromSource(vs, Test_Name).versionSpec()
v = spec.select(self.working_copy.availableVersions())
self.assertTrue(v)
def test_versionSpec(self):
vs = sourceparse.parseSourceURL(Test_Repo_With_Spec)
spec = git_access.GitComponent.createFromSource(
vs, Test_Name).versionSpec()
v = spec.select(self.working_copy.availableVersions())
self.assertTrue(v)
def test_hgURLs(self):
for url in HG_URLs:
sv = sourceparse.parseSourceURL(url)
self.assertEqual(sv.source_type, 'hg')
def test_gitURLs(self):
for url in Git_URLs:
sv = sourceparse.parseSourceURL(url)
self.assertEqual(sv.source_type, 'git')
def test_registryURLs(self):
for url in Registry_URLs:
sv = sourceparse.parseSourceURL(url)
self.assertEqual(sv.source_type, 'registry')
def displayOutdated(modules, dependency_specs, use_colours):
''' print information about outdated modules,
return 0 if there is nothing to be done and nonzero otherwise
'''
if use_colours:
DIM = colorama.Style.DIM #pylint: disable=no-member
NORMAL = colorama.Style.NORMAL #pylint: disable=no-member
BRIGHT = colorama.Style.BRIGHT #pylint: disable=no-member
YELLOW = colorama.Fore.YELLOW #pylint: disable=no-member
RED = colorama.Fore.RED #pylint: disable=no-member
GREEN = colorama.Fore.GREEN #pylint: disable=no-member
RESET = colorama.Style.RESET_ALL #pylint: disable=no-member
else:
DIM = BRIGHT = YELLOW = RED = GREEN = RESET = u''
status = 0
# access, , get components, internal
from yotta.lib import access
from yotta.lib import access_common
# sourceparse, , parse version source urls, internal
from yotta.lib import sourceparse
for name, m in modules.items():
if m.isTestDependency():
continue
try:
latest_v = access.latestSuitableVersion(name,
'*',
registry='modules',
quiet=True)
except access_common.Unavailable as e:
latest_v = None
if not m:
m_version = u' ' + RESET + BRIGHT + RED + u"missing" + RESET
else:
m_version = DIM + u'@%s' % (m.version)
if not latest_v:
print(u'%s%s%s%s not available from the registry%s' %
(RED, name, m_version, NORMAL, RESET))
status = 2
continue
elif not m or m.version < latest_v:
update_prevented_by = ''
if m:
specs_preventing_update = [
x for x in dependency_specs
if x.name == name and not sourceparse.parseSourceURL(
x.nonShrinkwrappedVersionReq()).semanticSpecMatches(
latest_v)
]
shrinkwrap_prevents_update = [
x for x in dependency_specs if x.name == name
and x.isShrinkwrapped() and not sourceparse.parseSourceURL(
x.versionReq()).semanticSpecMatches(latest_v)
]
if len(specs_preventing_update):
update_prevented_by = ' (update prevented by specifications: %s)' % (
', '.join([
'%s from %s' % (x.version_req, x.specifying_module)
for x in specs_preventing_update
]))
if len(shrinkwrap_prevents_update):
update_prevented_by += ' yotta-shrinkwrap.json prevents update'
if m.version.major() < latest_v.major():
# major versions being outdated might be deliberate, so not
# that bad:
colour = GREEN
elif m.version.minor() < latest_v.minor():
# minor outdated versions is moderately bad
colour = YELLOW
else:
# patch-outdated versions is really bad, because there should
# be no reason not to update:
colour = RED
else:
colour = RED
print(u'%s%s%s latest: %s%s%s%s' %
(name, m_version, RESET, colour, latest_v.version,
update_prevented_by, RESET))
if not status:
status = 1
return status
def displayOutdated(modules, dependency_specs, use_colours):
''' print information about outdated modules,
return 0 if there is nothing to be done and nonzero otherwise
'''
if use_colours:
DIM = colorama.Style.DIM #pylint: disable=no-member
NORMAL = colorama.Style.NORMAL #pylint: disable=no-member
BRIGHT = colorama.Style.BRIGHT #pylint: disable=no-member
YELLOW = colorama.Fore.YELLOW #pylint: disable=no-member
RED = colorama.Fore.RED #pylint: disable=no-member
GREEN = colorama.Fore.GREEN #pylint: disable=no-member
RESET = colorama.Style.RESET_ALL #pylint: disable=no-member
else:
DIM = BRIGHT = YELLOW = RED = GREEN = RESET = u''
status = 0
# access, , get components, internal
from yotta.lib import access
from yotta.lib import access_common
# sourceparse, , parse version source urls, internal
from yotta.lib import sourceparse
for name, m in modules.items():
if m.isTestDependency():
continue
try:
latest_v = access.latestSuitableVersion(name, '*', registry='modules', quiet=True)
except access_common.Unavailable as e:
latest_v = None
if not m:
m_version = u' ' + RESET + BRIGHT + RED + u"missing" + RESET
else:
m_version = DIM + u'@%s' % (m.version)
if not latest_v:
print(u'%s%s%s%s not available from the registry%s' % (RED, name, m_version, NORMAL, RESET))
status = 2
continue
elif not m or m.version < latest_v:
update_prevented_by = ''
if m:
specs_preventing_update = [
x for x in dependency_specs
if x.name == name and not
sourceparse.parseSourceURL(x.nonShrinkwrappedVersionReq()).semanticSpecMatches(latest_v)
]
shrinkwrap_prevents_update = [
x for x in dependency_specs
if x.name == name and x.isShrinkwrapped() and not
sourceparse.parseSourceURL(x.versionReq()).semanticSpecMatches(latest_v)
]
if len(specs_preventing_update):
update_prevented_by = ' (update prevented by specifications: %s)' % (
', '.join(['%s from %s' % (x.version_req, x.specifying_module) for x in specs_preventing_update])
)
if len(shrinkwrap_prevents_update):
update_prevented_by += ' yotta-shrinkwrap.json prevents update'
if m.version.major() < latest_v.major():
# major versions being outdated might be deliberate, so not
# that bad:
colour = GREEN
elif m.version.minor() < latest_v.minor():
# minor outdated versions is moderately bad
colour = YELLOW
else:
# patch-outdated versions is really bad, because there should
# be no reason not to update:
colour = RED
else:
colour = RED
print(u'%s%s%s latest: %s%s%s%s' % (name, m_version, RESET, colour, latest_v.version, update_prevented_by, RESET))
if not status:
status = 1
return status
def installComponentAsDependency(args, current_component):
logging.debug('install component %s as dependency of %s' %
(args.component, current_component))
if not current_component:
logging.debug(str(current_component.getError()))
logging.error('The current directory does not contain a valid module.')
return -1
target, errors = current_component.satisfyTarget(
args.target, additional_config=args.config)
if errors:
for error in errors:
logging.error(error)
return 1
modules_dir = current_component.modulesPath()
from yotta.lib import sourceparse
# check if we have both a name and specification
component_name, component_spec = sourceparse.parseModuleNameAndSpec(
args.component)
logging.info('%s, %s', component_name, component_spec)
if component_name == current_component.getName():
logging.error('will not install module %s as a dependency of itself',
component_name)
return -1
try:
installed = access.satisfyVersion(
component_name,
component_spec,
available={current_component.getName(): current_component},
search_paths=[modules_dir],
working_directory=modules_dir)
except access_common.AccessException as e:
logging.error(e)
return 1
# We always add the component to the dependencies of the current component
# (if it is not already present), and write that back to disk. Without
# writing to disk the dependency wouldn't be usable.
if installed and not current_component.hasDependency(component_name):
vs = sourceparse.parseSourceURL(component_spec)
if vs.source_type == 'registry':
saved_spec = current_component.saveDependency(installed)
else:
saved_spec = current_component.saveDependency(
installed, component_spec)
current_component.writeDescription()
logging.info('dependency %s: %s written to module.json',
component_name, saved_spec)
else:
logging.info('dependency %s is already present in module.json',
component_name)
# !!! should only install dependencies necessary for the one thing that
# we're installing (but existing components should be made available to
# satisfy dependencies)
components, errors = current_component.satisfyDependenciesRecursive(
target=target,
available_components=[(current_component.getName(), current_component)
],
test={
'own': 'toplevel',
'all': True,
'none': False
}[args.install_test_deps])
return checkPrintStatus(errors, components, current_component, target)
