示例#1
0
    def run(self, log, info):
        used_engines = []
        log.debug("All available engines: %s", info['ENGINES'])
        for engine in info['ENGINES'].split(" "):
            key = 'RUN' + engine.upper()
            if key in info and info[key] == 'True':
                used_engines.append(engine)
        log.debug("Effectively used engines: %s" % used_engines)

        if not isinstance(info[Keys.DATASET_CODE],list):
            info[Keys.DATASET_CODE] = [info[Keys.DATASET_CODE]]
        runs = len(info[Keys.DATASET_CODE])
        log.debug("Number of samples: %d" % runs)
        for i in range(runs):
            collectedconfig = {}
            for engine in used_engines:
                path = "%s.ini_%d" % (engine, i)
                if not os.path.exists(path):
                    raise RuntimeError("Required infofile not found " + path)
                else:
                    log.debug("Found infofile "+path)
                engineconfig = infohandler.get_handler(path).read(path)
                collectedconfig = dicts.merge(collectedconfig, engineconfig, priority='append')

            for key in collectedconfig.keys():
                if isinstance(collectedconfig[key], list):
                    collectedconfig[key] = dicts.unify(collectedconfig[key])

            collector_path = "%s_%d" % (info[Keys.MERGED], i)
            infohandler.get_handler(info[Keys.MERGED]).write(collectedconfig, collector_path)
            log.debug('Wrote outfile ' + collector_path)

        return info
示例#2
0
文件: merge.py 项目: lcb/applicake
    def run(self, log, info):
        paths = sorted(glob.glob(info[Keys.MERGE] + "_*"))

        #read in
        config_container = {}
        nofiles = len(paths)
        if nofiles == 0:
            raise RuntimeError("No files to merge found!")
        for path in paths:
            log.debug("Reading " + path)
            config = infohandler.get_handler(path).read(path)

            lastjob = config[Keys.SUBJOBLIST][-1]
            checksum = int(lastjob.split(Keys.SUBJOBSEP)[2])
            if nofiles != checksum:
                raise RuntimeError("Number of inputfiles %d and checksum %d do not match" % (nofiles, checksum))

            #append the current config to the ones with same parent subjobs
            parentjoblist = config[Keys.SUBJOBLIST][:-1]
            parentjobstr = self.parentjobs_to_str(parentjoblist)

            #remove one level from subjoblist
            config[Keys.SUBJOBLIST] = parentjoblist
            if not config[Keys.SUBJOBLIST]:
                del config[Keys.SUBJOBLIST]
            if parentjobstr in config_container:
                config_container[parentjobstr] = dicts.merge(config_container[parentjobstr], config, priority='append')
            else:
                config_container[parentjobstr] = config

        #unify (only possible after all collected)
        for config in config_container.values():
            for key in config.keys():
                if key == Keys.SUBJOBLIST:
                    config[key] = dicts.unify(config[key], unlist_single=False)
                    continue
                if isinstance(config[key], list):
                    config[key] = dicts.unify(config[key])

        #write back
        for i, config in enumerate(config_container.values()):
            path = info[Keys.MERGED] + '_' + str(i)
            log.debug("Writing out " + path)
            infohandler.get_handler(path).write(config, path)

        return info
示例#3
0
文件: collate.py 项目: lcb/applicake
    def run(self, log, info):
        ih = get_handler(info[Keys.COLLATE])
        paths = info[Keys.COLLATE].split(" ")
        del info[Keys.COLLATE]
        collector_config = info.copy()

        #read in
        for path in paths:
            log.debug('collating file [%s]' % path)
            config = ih.read(path)
            collector_config = dicts.merge(collector_config, config, priority='append')

        #unify
        for key in collector_config.keys():
            collector_config[key] = dicts.unify(collector_config[key])

        #write back
        return collector_config
示例#4
0
    def run(self, log, info):
        ih = get_handler(info[Keys.COLLATE])
        paths = info[Keys.COLLATE].split(" ")
        del info[Keys.COLLATE]
        collector_config = info.copy()

        #read in
        for path in paths:
            log.debug('collating file [%s]' % path)
            config = ih.read(path)
            collector_config = dicts.merge(collector_config,
                                           config,
                                           priority='append')

        #unify
        for key in collector_config.keys():
            collector_config[key] = dicts.unify(collector_config[key])

        #write back
        return collector_config