Esempio n. 1
0
def loadAdditionalConfig(config_path):
    ''' returns (error, config)
    '''
    error = None
    config = {}
    if not config_path:
        return (error, config)
    if os.path.isfile(config_path):
        try:
            config = ordered_json.load(config_path)
        except Exception as e:
            error = "Invalid syntax in file %s: %s" % (config_path, e)
    else:
        # try to interpret the argument as literal JSON
        try:
            config = ordered_json.loads(config_path)
        except Exception as e:
            # if this fails too, guess whether it was intended to be JSON or
            # not, and display an appropriate error message
            if '{' in config_path or '}' in config_path:
                error = "Invalid syntax in literal JSON: %s" % e
            else:
                error = "File \"%s\" does not exist" % config_path
    logger.debug('read additional config: %s', config)
    return (error, config)
Esempio n. 2
0
def loadAdditionalConfig(config_path):
    ''' returns (error, config)
    '''
    error = None
    config = {}
    if not config_path:
        return (error, config)
    if os.path.isfile(config_path):
        try:
            config = ordered_json.load(config_path)
        except Exception as e:
            error = "Invalid syntax in file %s: %s" % (config_path, e)
    else:
        # try to interpret the argument as literal JSON
        try:
            config = ordered_json.loads(config_path)
        except Exception as e:
            # if this fails too, guess whether it was intended to be JSON or
            # not, and display an appropriate error message
            if '{' in config_path or '}' in config_path:
                error = "Invalid syntax in literal JSON: %s" % e
            else:
                error = "File \"%s\" does not exist" % config_path
    logger.debug('read additional config: %s', config)
    return (error, config)
Esempio n. 3
0
 def read(self, filenames):
     '''' Read a list of files. Their configuration values are merged, with
          preference to values from files earlier in the list.
     '''
     for fn in filenames:
         try:
             self.configs[fn] = ordered_json.load(fn)
         except IOError:
             self.configs[fn] = OrderedDict()
Esempio n. 4
0
 def read(self, filenames):
     '''' Read a list of files. Their configuration values are merged, with
          preference to values from files earlier in the list.
     '''
     for fn in filenames:
         try:
             self.configs[fn] = ordered_json.load(fn)
         except IOError:
             self.configs[fn] = OrderedDict()
Esempio n. 5
0
File: pack.py Progetto: Timmmm/yotta
 def origin(self):
     ''' Read the .yotta_origin.json file (if present), and return the value
         of the 'url' property '''
     if self.origin_info is None:
         self.origin_info = {}
         try:
             self.origin_info = ordered_json.load(os.path.join(self.path, Origin_Info_Fname))
         except IOError:
             pass
     return self.origin_info.get('url', None)
Esempio n. 6
0
 def origin(self):
     ''' Read the .yotta_origin.json file (if present), and return the value
         of the 'url' property '''
     if self.origin_info is None:
         self.origin_info = {}
         try:
             self.origin_info = ordered_json.load(os.path.join(self.path, Origin_Info_Fname))
         except IOError:
             pass
     return self.origin_info.get('url', None)
Esempio n. 7
0
 def read(self, filenames):
     '''' Read a list of files. Their configuration values are merged, with
          preference to values from files earlier in the list.
     '''
     for fn in filenames:
         try:
             self.configs[fn] = ordered_json.load(fn)
         except IOError:
             self.configs[fn] = OrderedDict()
         except Exception as e:
             self.configs[fn] = OrderedDict()
             logging.warning(
                 "Failed to read settings file %s, it will be ignored. The error was: %s",
                 fn, e)
Esempio n. 8
0
def tryReadJSON(filename, schemaname):
    r = None
    try:
        with open(filename, 'r') as jsonfile:
            r = ordered_json.load(filename)
            if schemaname is not None:
                with open(schemaname, 'r') as schema_file:
                    schema = json.load(schema_file)
                    validator = jsonschema.Draft4Validator(schema)
                    for error in validator.iter_errors(r):
                        logger.error(
                            '%s is not valid under the schema: %s value %s',
                            filename, u'.'.join([str(x) for x in error.path]),
                            error.message)
    except IOError as e:
        if e.errno != errno.ENOENT:
            raise
    return r
Esempio n. 9
0
File: pack.py Progetto: Timmmm/yotta
def tryReadJSON(filename, schemaname):
    r = None
    try:
        with open(filename, 'r') as jsonfile:
            r = ordered_json.load(filename)
            if schemaname is not None:
                with open(schemaname, 'r') as schema_file:
                    schema = json.load(schema_file)
                    validator = jsonschema.Draft4Validator(schema)
                    for error in validator.iter_errors(r):
                        logger.error(
                            '%s is not valid under the schema: %s value %s',
                            filename,
                            u'.'.join([str(x) for x in error.path]),
                            error.message
                        )
    except IOError as e:
        if e.errno != errno.ENOENT:
            raise
    return r
Esempio n. 10
0
File: pack.py Progetto: Timmmm/yotta
    def __init__(
            self,
            path,
            description_filename,
            installed_linked,
            schema_filename = None,
            latest_suitable_version = None,
            inherit_shrinkwrap = None
        ):
        # version, , represent versions and specifications, internal
        from yotta.lib import version
        # vcs, , represent version controlled directories, internal
        from yotta.lib import vcs

        # resolve links at creation time, to minimise path lengths:
        self.unresolved_path = path
        self.path = fsutils.realpath(path)
        self.installed_linked = installed_linked
        self.vcs = None
        self.error = None
        self.latest_suitable_version = latest_suitable_version
        self.version = None
        self.description_filename = description_filename
        self.ignore_list_fname = Ignore_List_Fname
        self.ignore_patterns = copy.copy(Default_Publish_Ignore)
        self.origin_info = None
        description_file = os.path.join(path, description_filename)
        if os.path.isfile(description_file):
            try:
                self.description = ordered_json.load(description_file)
                if self.description:
                    if not 'name' in self.description:
                        raise Exception('missing "name"')
                    if 'version' in self.description:
                        self.version = version.Version(self.description['version'])
                    else:
                        raise Exception('missing "version"')
            except Exception as e:
                self.description = OrderedDict()
                self.error = "Description invalid %s: %s" % (description_file, e);
                logger.debug(self.error)
                raise InvalidDescription(self.error)
        else:
            self.error = "No %s file." % description_filename
            self.description = OrderedDict()
        try:
            with open(os.path.join(path, self.ignore_list_fname), 'r') as ignorefile:
                self.ignore_patterns += self._parseIgnoreFile(ignorefile)
        except IOError as e:
            if e.errno != errno.ENOENT:
                raise
        # warn about invalid yotta versions before schema errors (as new yotta
        # might introduce new schema)
        yotta_version_spec = None
        if self.description and self.description.get('yotta', None):
            try:
                yotta_version_spec = version.Spec(self.description['yotta'])
            except ValueError as e:
                logger.warning(
                    "could not parse yotta version spec '%s' from %s: it "+
                    "might require a newer version of yotta",
                    self.description['yotta'],
                    self.description['name']
                )
        if yotta_version_spec is not None:
            import yotta
            yotta_version = version.Version(yotta.__version__)
            if not yotta_version_spec.match(yotta_version):
                self.error = "requires yotta version %s (current version is %s). see http://yottadocs.mbed.com for update instructions" % (
                    str(yotta_version_spec),
                    str(yotta_version)
                )

        if self.description and schema_filename and not self.path in self.schema_errors_displayed:
            self.schema_errors_displayed.add(self.path)
            have_errors = False
            with open(schema_filename, 'r') as schema_file:
                schema = json.load(schema_file)
                validator = jsonschema.Draft4Validator(schema)
                for error in validator.iter_errors(self.description):
                    if not have_errors:
                        logger.warning(u'%s has invalid %s:' % (
                            os.path.split(self.path.rstrip('/'))[1],
                            description_filename
                        ))
                        have_errors = True
                    logger.warning(u"  %s value %s" % (u'.'.join([str(x) for x in error.path]), error.message))
            # for now schema validation errors aren't fatal... will be soon
            # though!
            #if have_errors:
            #    raise InvalidDescription('Invalid %s' % description_filename)
        self.inherited_shrinkwrap = None
        self.shrinkwrap = None
        # we can only apply shrinkwraps to instances with valid descriptions:
        # instances do not become valid after being invalid so this is safe
        # (but it means you cannot trust the shrinkwrap of an invalid
        # component)
        # (note that it is unsafe to use the __bool__ operator on self here as
        # we are not fully constructed)
        if self.description:
            self.inherited_shrinkwrap = inherit_shrinkwrap
            self.shrinkwrap = tryReadJSON(os.path.join(path, Shrinkwrap_Fname), Shrinkwrap_Schema)
            if self.shrinkwrap:
                logger.warning('dependencies of %s are pegged by yotta-shrinkwrap.json', self.getName())
                if self.inherited_shrinkwrap:
                    logger.warning('shrinkwrap in %s overrides inherited shrinkwrap', self.getName())
        #logger.info('%s created with inherited_shrinkwrap %s', self.getName(), self.inherited_shrinkwrap)
        self.vcs = vcs.getVCS(path)
Esempio n. 11
0
def getDerivedTarget(
        target_name_and_version,
                   targets_path,
                application_dir = None,
                install_missing = True,
               update_installed = False,
              additional_config = None,
                     shrinkwrap = None
    ):
    # access, , get components, internal
    from yotta.lib import access
    from yotta.lib import access_common
    ''' Get the specified target description, optionally ensuring that it (and
        all dependencies) are installed in targets_path.

        Returns (DerivedTarget, errors), or (None, errors) if the leaf target
        could not be found/installed.
    '''
    logger.debug('satisfy target: %s' % target_name_and_version);
    if ',' in target_name_and_version:
        name, version_req = target_name_and_version.split(',')
    else:
        name = target_name_and_version
        version_req = '*'

    # shrinkwrap is the raw json form, not mapping form here, so rearrange it
    # before indexing:
    if shrinkwrap is not None:
        shrinkwrap_version_req = {
            x['name']: x['version'] for x in shrinkwrap.get('targets', [])
        }.get(name, None)
    else:
        shrinkwrap_version_req = None
    if shrinkwrap_version_req is not None:
        logger.debug(
            'respecting shrinkwrap version %s for %s', shrinkwrap_version_req, name
        )

    dspec = pack.DependencySpec(
                                 name,
                                 version_req,
        shrinkwrap_version_req = shrinkwrap_version_req
    )

    leaf_target      = None
    previous_name    = dspec.name
    search_dirs      = [targets_path]
    target_hierarchy = []
    errors           = []
    while True:
        t = None
        try:
            if install_missing:
                t = access.satisfyVersion(
                                 name = dspec.name,
                     version_required = dspec.versionReq(),
                            available = target_hierarchy,
                         search_paths = search_dirs,
                    working_directory = targets_path,
                     update_installed = ('Update' if update_installed else None),
                                 type = 'target',
                   inherit_shrinkwrap = shrinkwrap
                )
            else:
                t = access.satisfyVersionFromSearchPaths(
                                 name = dspec.name,
                     version_required = dspec.versionReq(),
                         search_paths = search_dirs,
                                 type = 'target',
                   inherit_shrinkwrap = shrinkwrap
                )
        except access_common.AccessException as e:
            errors.append(e)
        if not t:
            if install_missing:
                logger.error(
                    'could not install target %s for %s' %
                    (dspec, previous_name)
                )
            break
        else:
            target_hierarchy.append(t)
            previous_name = dspec.name
            assert(isinstance(t, Target))
            dspec = t.baseTargetSpec() #pylint: disable=no-member
            if not leaf_target:
                leaf_target = t
            if dspec is None:
                break
    if leaf_target is None:
        return (None, errors)
    # if we have a valid target, try to load the app-specific config data (if
    # any):
    app_config = {}
    if application_dir is not None:
        app_config_fname = os.path.join(application_dir, App_Config_File)
        if os.path.exists(app_config_fname):
            try:
                app_config = ordered_json.load(app_config_fname)
            except Exception as e:
                errors.append(Exception("Invalid application config.json: %s" % (e)))
    return (DerivedTarget(leaf_target, target_hierarchy[1:], app_config, additional_config), errors)
Esempio n. 12
0
    def __init__(self,
                 path,
                 description_filename,
                 installed_linked,
                 schema_filename=None,
                 latest_suitable_version=None,
                 inherit_shrinkwrap=None):
        # version, , represent versions and specifications, internal
        from yotta.lib import version
        # vcs, , represent version controlled directories, internal
        from yotta.lib import vcs

        # resolve links at creation time, to minimise path lengths:
        self.unresolved_path = path
        self.path = fsutils.realpath(path)
        self.installed_linked = installed_linked
        self.vcs = None
        self.error = None
        self.latest_suitable_version = latest_suitable_version
        self.version = None
        self.description_filename = description_filename
        self.ignore_list_fname = Ignore_List_Fname
        self.ignore_patterns = copy.copy(Default_Publish_Ignore)
        self.origin_info = None
        description_file = os.path.join(path, description_filename)
        if os.path.isfile(description_file):
            try:
                self.description = ordered_json.load(description_file)
                if self.description:
                    if not 'name' in self.description:
                        raise Exception('missing "name"')
                    if 'version' in self.description:
                        self.version = version.Version(
                            self.description['version'])
                    else:
                        raise Exception('missing "version"')
            except Exception as e:
                self.description = OrderedDict()
                self.error = "Description invalid %s: %s" % (description_file,
                                                             e)
                logger.debug(self.error)
                raise InvalidDescription(self.error)
        else:
            self.error = "No %s file." % description_filename
            self.description = OrderedDict()
        try:
            with open(os.path.join(path, self.ignore_list_fname),
                      'r') as ignorefile:
                self.ignore_patterns += self._parseIgnoreFile(ignorefile)
        except IOError as e:
            if e.errno != errno.ENOENT:
                raise
        # warn about invalid yotta versions before schema errors (as new yotta
        # might introduce new schema)
        yotta_version_spec = None
        if self.description and self.description.get('yotta', None):
            try:
                yotta_version_spec = version.Spec(self.description['yotta'])
            except ValueError as e:
                logger.warning(
                    "could not parse yotta version spec '%s' from %s: it " +
                    "might require a newer version of yotta",
                    self.description['yotta'], self.description['name'])
        if yotta_version_spec is not None:
            import yotta
            yotta_version = version.Version(yotta.__version__)
            if not yotta_version_spec.match(yotta_version):
                self.error = "requires yotta version %s (current version is %s). see http://yottadocs.mbed.com for update instructions" % (
                    str(yotta_version_spec), str(yotta_version))

        if self.description and schema_filename and not self.path in self.schema_errors_displayed:
            self.schema_errors_displayed.add(self.path)
            have_errors = False
            with open(schema_filename, 'r') as schema_file:
                schema = json.load(schema_file)
                validator = jsonschema.Draft4Validator(schema)
                for error in validator.iter_errors(self.description):
                    if not have_errors:
                        logger.warning(u'%s has invalid %s:' % (os.path.split(
                            self.path.rstrip('/'))[1], description_filename))
                        have_errors = True
                    logger.warning(
                        u"  %s value %s" %
                        (u'.'.join([str(x)
                                    for x in error.path]), error.message))
            # for now schema validation errors aren't fatal... will be soon
            # though!
            #if have_errors:
            #    raise InvalidDescription('Invalid %s' % description_filename)
        self.inherited_shrinkwrap = None
        self.shrinkwrap = None
        # we can only apply shrinkwraps to instances with valid descriptions:
        # instances do not become valid after being invalid so this is safe
        # (but it means you cannot trust the shrinkwrap of an invalid
        # component)
        # (note that it is unsafe to use the __bool__ operator on self here as
        # we are not fully constructed)
        if self.description:
            if inherit_shrinkwrap is not None:
                # when inheriting a shrinkwrap, check that this module is
                # listed in the shrinkwrap, otherwise emit a warning:
                if next((x for x in inherit_shrinkwrap.get('modules', [])
                         if x['name'] == self.getName()), None) is None:
                    logger.warning("%s missing from shrinkwrap",
                                   self.getName())
                self.inherited_shrinkwrap = inherit_shrinkwrap
            self.shrinkwrap = tryReadJSON(os.path.join(path, Shrinkwrap_Fname),
                                          Shrinkwrap_Schema)
            if self.shrinkwrap:
                logger.warning(
                    'dependencies of %s are pegged by yotta-shrinkwrap.json',
                    self.getName())
                if self.inherited_shrinkwrap:
                    logger.warning(
                        'shrinkwrap in %s overrides inherited shrinkwrap',
                        self.getName())
        #logger.info('%s created with inherited_shrinkwrap %s', self.getName(), self.inherited_shrinkwrap)
        self.vcs = vcs.getVCS(path)
Esempio n. 13
0
def getDerivedTarget(target_name_and_version,
                     targets_path,
                     application_dir=None,
                     install_missing=True,
                     update_installed=False,
                     additional_config=None):
    # access, , get components, internal
    from yotta.lib import access
    from yotta.lib import access_common
    ''' Get the specified target description, optionally ensuring that it (and
        all dependencies) are installed in targets_path.

        Returns (DerivedTarget, errors), or (None, errors) if the leaf target
        could not be found/installed.
    '''
    logger.debug('satisfy target: %s' % target_name_and_version)
    if ',' in target_name_and_version:
        name, ver = target_name_and_version.split(',')
        dspec = pack.DependencySpec(name, ver)
    else:
        dspec = pack.DependencySpec(target_name_and_version, "*")

    leaf_target = None
    previous_name = dspec.name
    search_dirs = [targets_path]
    target_hierarchy = []
    errors = []
    while True:
        t = None
        try:
            if install_missing:
                t = access.satisfyVersion(
                    name=dspec.name,
                    version_required=dspec.versionReq(),
                    available=target_hierarchy,
                    search_paths=search_dirs,
                    working_directory=targets_path,
                    update_installed=('Update' if update_installed else None),
                    type='target')
            else:
                t = access.satisfyVersionFromSearchPaths(
                    name=dspec.name,
                    version_required=dspec.versionReq(),
                    search_paths=search_dirs,
                    type='target')
        except access_common.Unavailable as e:
            errors.append(e)
        if not t:
            if install_missing:
                logger.error('could not install target %s for %s' %
                             (dspec, previous_name))
            break
        else:
            target_hierarchy.append(t)
            previous_name = dspec.name
            assert (isinstance(t, Target))
            dspec = t.baseTargetSpec()  #pylint: disable=no-member
            if not leaf_target:
                leaf_target = t
            if dspec is None:
                break
    if leaf_target is None:
        return (None, errors)
    # if we have a valid target, try to load the app-specific config data (if
    # any):
    app_config = {}
    if application_dir is not None:
        app_config_fname = os.path.join(application_dir, App_Config_File)
        if os.path.exists(app_config_fname):
            try:
                app_config = ordered_json.load(app_config_fname)
            except Exception as e:
                errors.append(
                    Exception("Invalid application config.json: %s" % (e)))
    return (DerivedTarget(leaf_target, target_hierarchy[1:], app_config,
                          additional_config), errors)