def loadAdditionalConfig(config_path): ''' returns (error, config) ''' error = None config = {} if not config_path: return (error, config) if os.path.isfile(config_path): try: config = ordered_json.load(config_path) except Exception as e: error = "Invalid syntax in file %s: %s" % (config_path, e) else: # try to interpret the argument as literal JSON try: config = ordered_json.loads(config_path) except Exception as e: # if this fails too, guess whether it was intended to be JSON or # not, and display an appropriate error message if '{' in config_path or '}' in config_path: error = "Invalid syntax in literal JSON: %s" % e else: error = "File \"%s\" does not exist" % config_path logger.debug('read additional config: %s', config) return (error, config)
def read(self, filenames): '''' Read a list of files. Their configuration values are merged, with preference to values from files earlier in the list. ''' for fn in filenames: try: self.configs[fn] = ordered_json.load(fn) except IOError: self.configs[fn] = OrderedDict()
def origin(self): ''' Read the .yotta_origin.json file (if present), and return the value of the 'url' property ''' if self.origin_info is None: self.origin_info = {} try: self.origin_info = ordered_json.load(os.path.join(self.path, Origin_Info_Fname)) except IOError: pass return self.origin_info.get('url', None)
def read(self, filenames): '''' Read a list of files. Their configuration values are merged, with preference to values from files earlier in the list. ''' for fn in filenames: try: self.configs[fn] = ordered_json.load(fn) except IOError: self.configs[fn] = OrderedDict() except Exception as e: self.configs[fn] = OrderedDict() logging.warning( "Failed to read settings file %s, it will be ignored. The error was: %s", fn, e)
def tryReadJSON(filename, schemaname): r = None try: with open(filename, 'r') as jsonfile: r = ordered_json.load(filename) if schemaname is not None: with open(schemaname, 'r') as schema_file: schema = json.load(schema_file) validator = jsonschema.Draft4Validator(schema) for error in validator.iter_errors(r): logger.error( '%s is not valid under the schema: %s value %s', filename, u'.'.join([str(x) for x in error.path]), error.message) except IOError as e: if e.errno != errno.ENOENT: raise return r
def tryReadJSON(filename, schemaname): r = None try: with open(filename, 'r') as jsonfile: r = ordered_json.load(filename) if schemaname is not None: with open(schemaname, 'r') as schema_file: schema = json.load(schema_file) validator = jsonschema.Draft4Validator(schema) for error in validator.iter_errors(r): logger.error( '%s is not valid under the schema: %s value %s', filename, u'.'.join([str(x) for x in error.path]), error.message ) except IOError as e: if e.errno != errno.ENOENT: raise return r
def __init__( self, path, description_filename, installed_linked, schema_filename = None, latest_suitable_version = None, inherit_shrinkwrap = None ): # version, , represent versions and specifications, internal from yotta.lib import version # vcs, , represent version controlled directories, internal from yotta.lib import vcs # resolve links at creation time, to minimise path lengths: self.unresolved_path = path self.path = fsutils.realpath(path) self.installed_linked = installed_linked self.vcs = None self.error = None self.latest_suitable_version = latest_suitable_version self.version = None self.description_filename = description_filename self.ignore_list_fname = Ignore_List_Fname self.ignore_patterns = copy.copy(Default_Publish_Ignore) self.origin_info = None description_file = os.path.join(path, description_filename) if os.path.isfile(description_file): try: self.description = ordered_json.load(description_file) if self.description: if not 'name' in self.description: raise Exception('missing "name"') if 'version' in self.description: self.version = version.Version(self.description['version']) else: raise Exception('missing "version"') except Exception as e: self.description = OrderedDict() self.error = "Description invalid %s: %s" % (description_file, e); logger.debug(self.error) raise InvalidDescription(self.error) else: self.error = "No %s file." % description_filename self.description = OrderedDict() try: with open(os.path.join(path, self.ignore_list_fname), 'r') as ignorefile: self.ignore_patterns += self._parseIgnoreFile(ignorefile) except IOError as e: if e.errno != errno.ENOENT: raise # warn about invalid yotta versions before schema errors (as new yotta # might introduce new schema) yotta_version_spec = None if self.description and self.description.get('yotta', None): try: yotta_version_spec = version.Spec(self.description['yotta']) except ValueError as e: logger.warning( "could not parse yotta version spec '%s' from %s: it "+ "might require a newer version of yotta", self.description['yotta'], self.description['name'] ) if yotta_version_spec is not None: import yotta yotta_version = version.Version(yotta.__version__) if not yotta_version_spec.match(yotta_version): self.error = "requires yotta version %s (current version is %s). see http://yottadocs.mbed.com for update instructions" % ( str(yotta_version_spec), str(yotta_version) ) if self.description and schema_filename and not self.path in self.schema_errors_displayed: self.schema_errors_displayed.add(self.path) have_errors = False with open(schema_filename, 'r') as schema_file: schema = json.load(schema_file) validator = jsonschema.Draft4Validator(schema) for error in validator.iter_errors(self.description): if not have_errors: logger.warning(u'%s has invalid %s:' % ( os.path.split(self.path.rstrip('/'))[1], description_filename )) have_errors = True logger.warning(u" %s value %s" % (u'.'.join([str(x) for x in error.path]), error.message)) # for now schema validation errors aren't fatal... will be soon # though! #if have_errors: # raise InvalidDescription('Invalid %s' % description_filename) self.inherited_shrinkwrap = None self.shrinkwrap = None # we can only apply shrinkwraps to instances with valid descriptions: # instances do not become valid after being invalid so this is safe # (but it means you cannot trust the shrinkwrap of an invalid # component) # (note that it is unsafe to use the __bool__ operator on self here as # we are not fully constructed) if self.description: self.inherited_shrinkwrap = inherit_shrinkwrap self.shrinkwrap = tryReadJSON(os.path.join(path, Shrinkwrap_Fname), Shrinkwrap_Schema) if self.shrinkwrap: logger.warning('dependencies of %s are pegged by yotta-shrinkwrap.json', self.getName()) if self.inherited_shrinkwrap: logger.warning('shrinkwrap in %s overrides inherited shrinkwrap', self.getName()) #logger.info('%s created with inherited_shrinkwrap %s', self.getName(), self.inherited_shrinkwrap) self.vcs = vcs.getVCS(path)
def getDerivedTarget( target_name_and_version, targets_path, application_dir = None, install_missing = True, update_installed = False, additional_config = None, shrinkwrap = None ): # access, , get components, internal from yotta.lib import access from yotta.lib import access_common ''' Get the specified target description, optionally ensuring that it (and all dependencies) are installed in targets_path. Returns (DerivedTarget, errors), or (None, errors) if the leaf target could not be found/installed. ''' logger.debug('satisfy target: %s' % target_name_and_version); if ',' in target_name_and_version: name, version_req = target_name_and_version.split(',') else: name = target_name_and_version version_req = '*' # shrinkwrap is the raw json form, not mapping form here, so rearrange it # before indexing: if shrinkwrap is not None: shrinkwrap_version_req = { x['name']: x['version'] for x in shrinkwrap.get('targets', []) }.get(name, None) else: shrinkwrap_version_req = None if shrinkwrap_version_req is not None: logger.debug( 'respecting shrinkwrap version %s for %s', shrinkwrap_version_req, name ) dspec = pack.DependencySpec( name, version_req, shrinkwrap_version_req = shrinkwrap_version_req ) leaf_target = None previous_name = dspec.name search_dirs = [targets_path] target_hierarchy = [] errors = [] while True: t = None try: if install_missing: t = access.satisfyVersion( name = dspec.name, version_required = dspec.versionReq(), available = target_hierarchy, search_paths = search_dirs, working_directory = targets_path, update_installed = ('Update' if update_installed else None), type = 'target', inherit_shrinkwrap = shrinkwrap ) else: t = access.satisfyVersionFromSearchPaths( name = dspec.name, version_required = dspec.versionReq(), search_paths = search_dirs, type = 'target', inherit_shrinkwrap = shrinkwrap ) except access_common.AccessException as e: errors.append(e) if not t: if install_missing: logger.error( 'could not install target %s for %s' % (dspec, previous_name) ) break else: target_hierarchy.append(t) previous_name = dspec.name assert(isinstance(t, Target)) dspec = t.baseTargetSpec() #pylint: disable=no-member if not leaf_target: leaf_target = t if dspec is None: break if leaf_target is None: return (None, errors) # if we have a valid target, try to load the app-specific config data (if # any): app_config = {} if application_dir is not None: app_config_fname = os.path.join(application_dir, App_Config_File) if os.path.exists(app_config_fname): try: app_config = ordered_json.load(app_config_fname) except Exception as e: errors.append(Exception("Invalid application config.json: %s" % (e))) return (DerivedTarget(leaf_target, target_hierarchy[1:], app_config, additional_config), errors)
def __init__(self, path, description_filename, installed_linked, schema_filename=None, latest_suitable_version=None, inherit_shrinkwrap=None): # version, , represent versions and specifications, internal from yotta.lib import version # vcs, , represent version controlled directories, internal from yotta.lib import vcs # resolve links at creation time, to minimise path lengths: self.unresolved_path = path self.path = fsutils.realpath(path) self.installed_linked = installed_linked self.vcs = None self.error = None self.latest_suitable_version = latest_suitable_version self.version = None self.description_filename = description_filename self.ignore_list_fname = Ignore_List_Fname self.ignore_patterns = copy.copy(Default_Publish_Ignore) self.origin_info = None description_file = os.path.join(path, description_filename) if os.path.isfile(description_file): try: self.description = ordered_json.load(description_file) if self.description: if not 'name' in self.description: raise Exception('missing "name"') if 'version' in self.description: self.version = version.Version( self.description['version']) else: raise Exception('missing "version"') except Exception as e: self.description = OrderedDict() self.error = "Description invalid %s: %s" % (description_file, e) logger.debug(self.error) raise InvalidDescription(self.error) else: self.error = "No %s file." % description_filename self.description = OrderedDict() try: with open(os.path.join(path, self.ignore_list_fname), 'r') as ignorefile: self.ignore_patterns += self._parseIgnoreFile(ignorefile) except IOError as e: if e.errno != errno.ENOENT: raise # warn about invalid yotta versions before schema errors (as new yotta # might introduce new schema) yotta_version_spec = None if self.description and self.description.get('yotta', None): try: yotta_version_spec = version.Spec(self.description['yotta']) except ValueError as e: logger.warning( "could not parse yotta version spec '%s' from %s: it " + "might require a newer version of yotta", self.description['yotta'], self.description['name']) if yotta_version_spec is not None: import yotta yotta_version = version.Version(yotta.__version__) if not yotta_version_spec.match(yotta_version): self.error = "requires yotta version %s (current version is %s). see http://yottadocs.mbed.com for update instructions" % ( str(yotta_version_spec), str(yotta_version)) if self.description and schema_filename and not self.path in self.schema_errors_displayed: self.schema_errors_displayed.add(self.path) have_errors = False with open(schema_filename, 'r') as schema_file: schema = json.load(schema_file) validator = jsonschema.Draft4Validator(schema) for error in validator.iter_errors(self.description): if not have_errors: logger.warning(u'%s has invalid %s:' % (os.path.split( self.path.rstrip('/'))[1], description_filename)) have_errors = True logger.warning( u" %s value %s" % (u'.'.join([str(x) for x in error.path]), error.message)) # for now schema validation errors aren't fatal... will be soon # though! #if have_errors: # raise InvalidDescription('Invalid %s' % description_filename) self.inherited_shrinkwrap = None self.shrinkwrap = None # we can only apply shrinkwraps to instances with valid descriptions: # instances do not become valid after being invalid so this is safe # (but it means you cannot trust the shrinkwrap of an invalid # component) # (note that it is unsafe to use the __bool__ operator on self here as # we are not fully constructed) if self.description: if inherit_shrinkwrap is not None: # when inheriting a shrinkwrap, check that this module is # listed in the shrinkwrap, otherwise emit a warning: if next((x for x in inherit_shrinkwrap.get('modules', []) if x['name'] == self.getName()), None) is None: logger.warning("%s missing from shrinkwrap", self.getName()) self.inherited_shrinkwrap = inherit_shrinkwrap self.shrinkwrap = tryReadJSON(os.path.join(path, Shrinkwrap_Fname), Shrinkwrap_Schema) if self.shrinkwrap: logger.warning( 'dependencies of %s are pegged by yotta-shrinkwrap.json', self.getName()) if self.inherited_shrinkwrap: logger.warning( 'shrinkwrap in %s overrides inherited shrinkwrap', self.getName()) #logger.info('%s created with inherited_shrinkwrap %s', self.getName(), self.inherited_shrinkwrap) self.vcs = vcs.getVCS(path)
def getDerivedTarget(target_name_and_version, targets_path, application_dir=None, install_missing=True, update_installed=False, additional_config=None): # access, , get components, internal from yotta.lib import access from yotta.lib import access_common ''' Get the specified target description, optionally ensuring that it (and all dependencies) are installed in targets_path. Returns (DerivedTarget, errors), or (None, errors) if the leaf target could not be found/installed. ''' logger.debug('satisfy target: %s' % target_name_and_version) if ',' in target_name_and_version: name, ver = target_name_and_version.split(',') dspec = pack.DependencySpec(name, ver) else: dspec = pack.DependencySpec(target_name_and_version, "*") leaf_target = None previous_name = dspec.name search_dirs = [targets_path] target_hierarchy = [] errors = [] while True: t = None try: if install_missing: t = access.satisfyVersion( name=dspec.name, version_required=dspec.versionReq(), available=target_hierarchy, search_paths=search_dirs, working_directory=targets_path, update_installed=('Update' if update_installed else None), type='target') else: t = access.satisfyVersionFromSearchPaths( name=dspec.name, version_required=dspec.versionReq(), search_paths=search_dirs, type='target') except access_common.Unavailable as e: errors.append(e) if not t: if install_missing: logger.error('could not install target %s for %s' % (dspec, previous_name)) break else: target_hierarchy.append(t) previous_name = dspec.name assert (isinstance(t, Target)) dspec = t.baseTargetSpec() #pylint: disable=no-member if not leaf_target: leaf_target = t if dspec is None: break if leaf_target is None: return (None, errors) # if we have a valid target, try to load the app-specific config data (if # any): app_config = {} if application_dir is not None: app_config_fname = os.path.join(application_dir, App_Config_File) if os.path.exists(app_config_fname): try: app_config = ordered_json.load(app_config_fname) except Exception as e: errors.append( Exception("Invalid application config.json: %s" % (e))) return (DerivedTarget(leaf_target, target_hierarchy[1:], app_config, additional_config), errors)