def loadAdditionalConfig(config_path):
''' returns (error, config)
'''
error = None
config = {}
if not config_path:
return (error, config)
if os.path.isfile(config_path):
try:
config = ordered_json.load(config_path)
except Exception as e:
error = "Invalid syntax in file %s: %s" % (config_path, e)
else:
# try to interpret the argument as literal JSON
try:
config = ordered_json.loads(config_path)
except Exception as e:
# if this fails too, guess whether it was intended to be JSON or
# not, and display an appropriate error message
if '{' in config_path or '}' in config_path:
error = "Invalid syntax in literal JSON: %s" % e
else:
error = "File \"%s\" does not exist" % config_path
logger.debug('read additional config: %s', config)
return (error, config)
def loadAdditionalConfig(config_path):
''' returns (error, config)
'''
error = None
config = {}
if not config_path:
return (error, config)
if os.path.isfile(config_path):
try:
config = ordered_json.load(config_path)
except Exception as e:
error = "Invalid syntax in file %s: %s" % (config_path, e)
else:
# try to interpret the argument as literal JSON
try:
config = ordered_json.loads(config_path)
except Exception as e:
# if this fails too, guess whether it was intended to be JSON or
# not, and display an appropriate error message
if '{' in config_path or '}' in config_path:
error = "Invalid syntax in literal JSON: %s" % e
else:
error = "File \"%s\" does not exist" % config_path
logger.debug('read additional config: %s', config)
return (error, config)
def read(self, filenames):
'''' Read a list of files. Their configuration values are merged, with
preference to values from files earlier in the list.
'''
for fn in filenames:
try:
self.configs[fn] = ordered_json.load(fn)
except IOError:
self.configs[fn] = OrderedDict()
def read(self, filenames):
'''' Read a list of files. Their configuration values are merged, with
preference to values from files earlier in the list.
'''
for fn in filenames:
try:
self.configs[fn] = ordered_json.load(fn)
except IOError:
self.configs[fn] = OrderedDict()
def origin(self):
''' Read the .yotta_origin.json file (if present), and return the value
of the 'url' property '''
if self.origin_info is None:
self.origin_info = {}
try:
self.origin_info = ordered_json.load(os.path.join(self.path, Origin_Info_Fname))
except IOError:
pass
return self.origin_info.get('url', None)
def origin(self):
''' Read the .yotta_origin.json file (if present), and return the value
of the 'url' property '''
if self.origin_info is None:
self.origin_info = {}
try:
self.origin_info = ordered_json.load(os.path.join(self.path, Origin_Info_Fname))
except IOError:
pass
return self.origin_info.get('url', None)
def read(self, filenames):
'''' Read a list of files. Their configuration values are merged, with
preference to values from files earlier in the list.
'''
for fn in filenames:
try:
self.configs[fn] = ordered_json.load(fn)
except IOError:
self.configs[fn] = OrderedDict()
except Exception as e:
self.configs[fn] = OrderedDict()
logging.warning(
"Failed to read settings file %s, it will be ignored. The error was: %s",
fn, e)
def tryReadJSON(filename, schemaname):
r = None
try:
with open(filename, 'r') as jsonfile:
r = ordered_json.load(filename)
if schemaname is not None:
with open(schemaname, 'r') as schema_file:
schema = json.load(schema_file)
validator = jsonschema.Draft4Validator(schema)
for error in validator.iter_errors(r):
logger.error(
'%s is not valid under the schema: %s value %s',
filename, u'.'.join([str(x) for x in error.path]),
error.message)
except IOError as e:
if e.errno != errno.ENOENT:
raise
return r
def tryReadJSON(filename, schemaname):
r = None
try:
with open(filename, 'r') as jsonfile:
r = ordered_json.load(filename)
if schemaname is not None:
with open(schemaname, 'r') as schema_file:
schema = json.load(schema_file)
validator = jsonschema.Draft4Validator(schema)
for error in validator.iter_errors(r):
logger.error(
'%s is not valid under the schema: %s value %s',
filename,
u'.'.join([str(x) for x in error.path]),
error.message
)
except IOError as e:
if e.errno != errno.ENOENT:
raise
return r
def __init__(
self,
path,
description_filename,
installed_linked,
schema_filename = None,
latest_suitable_version = None,
inherit_shrinkwrap = None
):
# version, , represent versions and specifications, internal
from yotta.lib import version
# vcs, , represent version controlled directories, internal
from yotta.lib import vcs
# resolve links at creation time, to minimise path lengths:
self.unresolved_path = path
self.path = fsutils.realpath(path)
self.installed_linked = installed_linked
self.vcs = None
self.error = None
self.latest_suitable_version = latest_suitable_version
self.version = None
self.description_filename = description_filename
self.ignore_list_fname = Ignore_List_Fname
self.ignore_patterns = copy.copy(Default_Publish_Ignore)
self.origin_info = None
description_file = os.path.join(path, description_filename)
if os.path.isfile(description_file):
try:
self.description = ordered_json.load(description_file)
if self.description:
if not 'name' in self.description:
raise Exception('missing "name"')
if 'version' in self.description:
self.version = version.Version(self.description['version'])
else:
raise Exception('missing "version"')
except Exception as e:
self.description = OrderedDict()
self.error = "Description invalid %s: %s" % (description_file, e);
logger.debug(self.error)
raise InvalidDescription(self.error)
else:
self.error = "No %s file." % description_filename
self.description = OrderedDict()
try:
with open(os.path.join(path, self.ignore_list_fname), 'r') as ignorefile:
self.ignore_patterns += self._parseIgnoreFile(ignorefile)
except IOError as e:
if e.errno != errno.ENOENT:
raise
# warn about invalid yotta versions before schema errors (as new yotta
# might introduce new schema)
yotta_version_spec = None
if self.description and self.description.get('yotta', None):
try:
yotta_version_spec = version.Spec(self.description['yotta'])
except ValueError as e:
logger.warning(
"could not parse yotta version spec '%s' from %s: it "+
"might require a newer version of yotta",
self.description['yotta'],
self.description['name']
)
if yotta_version_spec is not None:
import yotta
yotta_version = version.Version(yotta.__version__)
if not yotta_version_spec.match(yotta_version):
self.error = "requires yotta version %s (current version is %s). see http://yottadocs.mbed.com for update instructions" % (
str(yotta_version_spec),
str(yotta_version)
)
if self.description and schema_filename and not self.path in self.schema_errors_displayed:
self.schema_errors_displayed.add(self.path)
have_errors = False
with open(schema_filename, 'r') as schema_file:
schema = json.load(schema_file)
validator = jsonschema.Draft4Validator(schema)
for error in validator.iter_errors(self.description):
if not have_errors:
logger.warning(u'%s has invalid %s:' % (
os.path.split(self.path.rstrip('/'))[1],
description_filename
))
have_errors = True
logger.warning(u" %s value %s" % (u'.'.join([str(x) for x in error.path]), error.message))
# for now schema validation errors aren't fatal... will be soon
# though!
#if have_errors:
# raise InvalidDescription('Invalid %s' % description_filename)
self.inherited_shrinkwrap = None
self.shrinkwrap = None
# we can only apply shrinkwraps to instances with valid descriptions:
# instances do not become valid after being invalid so this is safe
# (but it means you cannot trust the shrinkwrap of an invalid
# component)
# (note that it is unsafe to use the __bool__ operator on self here as
# we are not fully constructed)
if self.description:
self.inherited_shrinkwrap = inherit_shrinkwrap
self.shrinkwrap = tryReadJSON(os.path.join(path, Shrinkwrap_Fname), Shrinkwrap_Schema)
if self.shrinkwrap:
logger.warning('dependencies of %s are pegged by yotta-shrinkwrap.json', self.getName())
if self.inherited_shrinkwrap:
logger.warning('shrinkwrap in %s overrides inherited shrinkwrap', self.getName())
#logger.info('%s created with inherited_shrinkwrap %s', self.getName(), self.inherited_shrinkwrap)
self.vcs = vcs.getVCS(path)
def getDerivedTarget(
target_name_and_version,
targets_path,
application_dir = None,
install_missing = True,
update_installed = False,
additional_config = None,
shrinkwrap = None
):
# access, , get components, internal
from yotta.lib import access
from yotta.lib import access_common
''' Get the specified target description, optionally ensuring that it (and
all dependencies) are installed in targets_path.
Returns (DerivedTarget, errors), or (None, errors) if the leaf target
could not be found/installed.
'''
logger.debug('satisfy target: %s' % target_name_and_version);
if ',' in target_name_and_version:
name, version_req = target_name_and_version.split(',')
else:
name = target_name_and_version
version_req = '*'
# shrinkwrap is the raw json form, not mapping form here, so rearrange it
# before indexing:
if shrinkwrap is not None:
shrinkwrap_version_req = {
x['name']: x['version'] for x in shrinkwrap.get('targets', [])
}.get(name, None)
else:
shrinkwrap_version_req = None
if shrinkwrap_version_req is not None:
logger.debug(
'respecting shrinkwrap version %s for %s', shrinkwrap_version_req, name
)
dspec = pack.DependencySpec(
name,
version_req,
shrinkwrap_version_req = shrinkwrap_version_req
)
leaf_target = None
previous_name = dspec.name
search_dirs = [targets_path]
target_hierarchy = []
errors = []
while True:
t = None
try:
if install_missing:
t = access.satisfyVersion(
name = dspec.name,
version_required = dspec.versionReq(),
available = target_hierarchy,
search_paths = search_dirs,
working_directory = targets_path,
update_installed = ('Update' if update_installed else None),
type = 'target',
inherit_shrinkwrap = shrinkwrap
)
else:
t = access.satisfyVersionFromSearchPaths(
name = dspec.name,
version_required = dspec.versionReq(),
search_paths = search_dirs,
type = 'target',
inherit_shrinkwrap = shrinkwrap
)
except access_common.AccessException as e:
errors.append(e)
if not t:
if install_missing:
logger.error(
'could not install target %s for %s' %
(dspec, previous_name)
)
break
else:
target_hierarchy.append(t)
previous_name = dspec.name
assert(isinstance(t, Target))
dspec = t.baseTargetSpec() #pylint: disable=no-member
if not leaf_target:
leaf_target = t
if dspec is None:
break
if leaf_target is None:
return (None, errors)
# if we have a valid target, try to load the app-specific config data (if
# any):
app_config = {}
if application_dir is not None:
app_config_fname = os.path.join(application_dir, App_Config_File)
if os.path.exists(app_config_fname):
try:
app_config = ordered_json.load(app_config_fname)
except Exception as e:
errors.append(Exception("Invalid application config.json: %s" % (e)))
return (DerivedTarget(leaf_target, target_hierarchy[1:], app_config, additional_config), errors)
def __init__(self,
path,
description_filename,
installed_linked,
schema_filename=None,
latest_suitable_version=None,
inherit_shrinkwrap=None):
# version, , represent versions and specifications, internal
from yotta.lib import version
# vcs, , represent version controlled directories, internal
from yotta.lib import vcs
# resolve links at creation time, to minimise path lengths:
self.unresolved_path = path
self.path = fsutils.realpath(path)
self.installed_linked = installed_linked
self.vcs = None
self.error = None
self.latest_suitable_version = latest_suitable_version
self.version = None
self.description_filename = description_filename
self.ignore_list_fname = Ignore_List_Fname
self.ignore_patterns = copy.copy(Default_Publish_Ignore)
self.origin_info = None
description_file = os.path.join(path, description_filename)
if os.path.isfile(description_file):
try:
self.description = ordered_json.load(description_file)
if self.description:
if not 'name' in self.description:
raise Exception('missing "name"')
if 'version' in self.description:
self.version = version.Version(
self.description['version'])
else:
raise Exception('missing "version"')
except Exception as e:
self.description = OrderedDict()
self.error = "Description invalid %s: %s" % (description_file,
e)
logger.debug(self.error)
raise InvalidDescription(self.error)
else:
self.error = "No %s file." % description_filename
self.description = OrderedDict()
try:
with open(os.path.join(path, self.ignore_list_fname),
'r') as ignorefile:
self.ignore_patterns += self._parseIgnoreFile(ignorefile)
except IOError as e:
if e.errno != errno.ENOENT:
raise
# warn about invalid yotta versions before schema errors (as new yotta
# might introduce new schema)
yotta_version_spec = None
if self.description and self.description.get('yotta', None):
try:
yotta_version_spec = version.Spec(self.description['yotta'])
except ValueError as e:
logger.warning(
"could not parse yotta version spec '%s' from %s: it " +
"might require a newer version of yotta",
self.description['yotta'], self.description['name'])
if yotta_version_spec is not None:
import yotta
yotta_version = version.Version(yotta.__version__)
if not yotta_version_spec.match(yotta_version):
self.error = "requires yotta version %s (current version is %s). see http://yottadocs.mbed.com for update instructions" % (
str(yotta_version_spec), str(yotta_version))
if self.description and schema_filename and not self.path in self.schema_errors_displayed:
self.schema_errors_displayed.add(self.path)
have_errors = False
with open(schema_filename, 'r') as schema_file:
schema = json.load(schema_file)
validator = jsonschema.Draft4Validator(schema)
for error in validator.iter_errors(self.description):
if not have_errors:
logger.warning(u'%s has invalid %s:' % (os.path.split(
self.path.rstrip('/'))[1], description_filename))
have_errors = True
logger.warning(
u" %s value %s" %
(u'.'.join([str(x)
for x in error.path]), error.message))
# for now schema validation errors aren't fatal... will be soon
# though!
#if have_errors:
# raise InvalidDescription('Invalid %s' % description_filename)
self.inherited_shrinkwrap = None
self.shrinkwrap = None
# we can only apply shrinkwraps to instances with valid descriptions:
# instances do not become valid after being invalid so this is safe
# (but it means you cannot trust the shrinkwrap of an invalid
# component)
# (note that it is unsafe to use the __bool__ operator on self here as
# we are not fully constructed)
if self.description:
if inherit_shrinkwrap is not None:
# when inheriting a shrinkwrap, check that this module is
# listed in the shrinkwrap, otherwise emit a warning:
if next((x for x in inherit_shrinkwrap.get('modules', [])
if x['name'] == self.getName()), None) is None:
logger.warning("%s missing from shrinkwrap",
self.getName())
self.inherited_shrinkwrap = inherit_shrinkwrap
self.shrinkwrap = tryReadJSON(os.path.join(path, Shrinkwrap_Fname),
Shrinkwrap_Schema)
if self.shrinkwrap:
logger.warning(
'dependencies of %s are pegged by yotta-shrinkwrap.json',
self.getName())
if self.inherited_shrinkwrap:
logger.warning(
'shrinkwrap in %s overrides inherited shrinkwrap',
self.getName())
#logger.info('%s created with inherited_shrinkwrap %s', self.getName(), self.inherited_shrinkwrap)
self.vcs = vcs.getVCS(path)
def getDerivedTarget(target_name_and_version,
targets_path,
application_dir=None,
install_missing=True,
update_installed=False,
additional_config=None):
# access, , get components, internal
from yotta.lib import access
from yotta.lib import access_common
''' Get the specified target description, optionally ensuring that it (and
all dependencies) are installed in targets_path.
Returns (DerivedTarget, errors), or (None, errors) if the leaf target
could not be found/installed.
'''
logger.debug('satisfy target: %s' % target_name_and_version)
if ',' in target_name_and_version:
name, ver = target_name_and_version.split(',')
dspec = pack.DependencySpec(name, ver)
else:
dspec = pack.DependencySpec(target_name_and_version, "*")
leaf_target = None
previous_name = dspec.name
search_dirs = [targets_path]
target_hierarchy = []
errors = []
while True:
t = None
try:
if install_missing:
t = access.satisfyVersion(
name=dspec.name,
version_required=dspec.versionReq(),
available=target_hierarchy,
search_paths=search_dirs,
working_directory=targets_path,
update_installed=('Update' if update_installed else None),
type='target')
else:
t = access.satisfyVersionFromSearchPaths(
name=dspec.name,
version_required=dspec.versionReq(),
search_paths=search_dirs,
type='target')
except access_common.Unavailable as e:
errors.append(e)
if not t:
if install_missing:
logger.error('could not install target %s for %s' %
(dspec, previous_name))
break
else:
target_hierarchy.append(t)
previous_name = dspec.name
assert (isinstance(t, Target))
dspec = t.baseTargetSpec() #pylint: disable=no-member
if not leaf_target:
leaf_target = t
if dspec is None:
break
if leaf_target is None:
return (None, errors)
# if we have a valid target, try to load the app-specific config data (if
# any):
app_config = {}
if application_dir is not None:
app_config_fname = os.path.join(application_dir, App_Config_File)
if os.path.exists(app_config_fname):
try:
app_config = ordered_json.load(app_config_fname)
except Exception as e:
errors.append(
Exception("Invalid application config.json: %s" % (e)))
return (DerivedTarget(leaf_target, target_hierarchy[1:], app_config,
additional_config), errors)