Esempio n. 1
0
def merge_params(default_params, override_params):

    merger = ConfigMerger(resolver=overwriteMergeResolve);
    params = Config();
    merger.merge(params, default_params);
    merger.merge(params, override_params);
    return params;
Esempio n. 2
0
def showRavenor(ui):
    sys.path.append('../Ravenor')
    import raven
    from winterstone.base import WinterConfig
    from config import ConfigMerger

    raven.API().CWD+='../Ravenor/'
    ravenconfig=WinterConfig(file(raven.API().CWD  + 'config/main.cfg'))
    darniconfig=WinterConfig(file(CWD  + 'config/main.cfg'))
    merger = ConfigMerger(resolve)
    merger.merge(darniconfig.options.app, ravenconfig.options.app)
    API().config=darniconfig
    from etherstone.base import EtherIntegration
    wi=EtherIntegration(ui,UI=False)
    ravenor=raven.Panel(ui)
    ravenor.tree=raven.Tree(ravenor,ravenor.api.CWD)
    newpanel=raven.NewForm(ravenor)
    ui.sideBar = WinterSideBar(ui)
    ui.createSBAction('squares','Content',ravenor.tree.getWidget(),toolbar=True,keyseq=darniconfig.options.app.tree_shortcut)
    ui.createSBAction('newPage','New',newpanel,toolbar=True,keyseq=darniconfig.options.app.new_shortcut)
    ui.createSBAction('newPage','Edit',newpanel,toolbar=False,keyseq=darniconfig.options.app.new_shortcut)

    cmd=raven.CommandLine(ravenor)
    lay=QWidget()
    lay.setLayout(QVBoxLayout())
    lay.layout().addWidget(cmd)
    lay.layout().addWidget(ravenor)

    
    ui.mainWidget.addTab(lay,QIcon(raven.API().CWD+'icons/app.png'),'Ravenor')
Esempio n. 3
0
def initialize(load_test_conf=False):
    global _CONFIG
    _merger = ConfigMerger(overwriteMergeResolve)
    conf_list = []
    test_conf_list = []

    for path in sys.path:
        file_list = glob.glob(os.path.join(path, "env*.conf"))
        for file_path in file_list:
            if file_path.find("_test.conf") >= 0:
                test_conf_list.append(file_path)
            else:
                conf_list.append(file_path)

    if load_test_conf:
        all_conf_list = conf_list + test_conf_list
    else:
        all_conf_list = conf_list

    _logger.debug("load config files %s", all_conf_list)
    for file_path in all_conf_list:
        f = file(file_path)

        try:
            cfg = Config(f)
            _merger.merge(_CONFIG, cfg)
        except:
            _logger.warn("load config file %s failed.", file_path, exc_info=True)

    _logger.debug("all configurations: %s", _CONFIG)
Esempio n. 4
0
def merge_params(default_params, override_params):
    merger = ConfigMerger(resolver=overwriteMergeResolve)
    params = Config()
    if default_params is not None:
        merger.merge(params, default_params)
    if override_params is not None:
        merger.merge(params, override_params)
    return params
Esempio n. 5
0
def merge_params(default_params, override_params):
    merger = ConfigMerger(resolver=overwriteMergeResolve)
    params = Config()
    if default_params is not None:
        merger.merge(params, default_params)
    if override_params is not None:
        merger.merge(params, override_params)
    return params
Esempio n. 6
0
 def load(self, config_name):
     cfg = Config(file(os.path.join(self.BASE_PATH, "base.cfg")))
     if not config_name.endswith(".cfg"):
         config_name = config_name + ".cfg"
     cfg1 = Config(file(config_name))
     self.sections_to_merge = cfg.sections_to_merge
     merger = ConfigMerger(self.merge_by_overwrite)
     merger.merge(cfg, cfg1)
     return cfg
Esempio n. 7
0
def load(_reload=False):
    global THE_CONF
    if _reload or THE_CONF is None:
        THE_CONF = Config()
        THE_CONF.load(file(CONF_FILE_COMMON))
        THE_CONF.load(file(CONF_FILE_ENV))
        if CONF_FILE_LOCAL is not None:
            local_conf = Config()
            merger = ConfigMerger(_local_resolver)
            local_conf.load(file(CONF_FILE_LOCAL))
            merger.merge(THE_CONF, local_conf)
Esempio n. 8
0
def test_config():
    cfg = Config(file("test_base.cfg"))
    cfg2 = Config(file("test_override.cfg"))
    merger = ConfigMerger(mergeByOverwrite)
    merger.merge(cfg, cfg2)
    assert cfg.train.mid_layer_sizes[0] == 99, "midlayer_sizes[0] should take value from override (99)"
    assert cfg.train.inner_code_length == 300, "inner_code_length should be as defined in base (300)"
    assert cfg.val1 == 20, "val1 should be as defined in override (10)"
    assert (
        cfg.train.n_ins == 20
    ), "n_ins should very cleverly combine the formula from base with value from override to give 20"
Esempio n. 9
0
def showRavenor(ui):
    sys.path.append('../Ravenor')
    import raven
    from winterstone.base import WinterConfig
    from config import ConfigMerger

    raven.API().CWD += '../Ravenor/'
    ravenconfig = WinterConfig(file(raven.API().CWD + 'config/main.cfg'))
    darniconfig = WinterConfig(file(CWD + 'config/main.cfg'))
    merger = ConfigMerger(resolve)
    merger.merge(darniconfig.options.app, ravenconfig.options.app)
    API().config = darniconfig
    from etherstone.base import EtherIntegration

    wi = EtherIntegration(ui, UI=False)
    ravenor = raven.Panel(ui)
    ravenor.tree = raven.Tree(ravenor, ravenor.api.CWD)
    newpanel = raven.NewForm(ravenor)
    ui.sideBar = WinterSideBar(ui)
    ui.createSBAction('squares',
                      'Content',
                      ravenor.tree.getWidget(),
                      toolbar=True,
                      keyseq=darniconfig.options.app.tree_shortcut)
    ui.createSBAction('newPage',
                      'New',
                      newpanel,
                      toolbar=True,
                      keyseq=darniconfig.options.app.new_shortcut)
    ui.createSBAction('newPage',
                      'Edit',
                      newpanel,
                      toolbar=False,
                      keyseq=darniconfig.options.app.new_shortcut)

    cmd = raven.CommandLine(ravenor)
    lay = QWidget()
    lay.setLayout(QVBoxLayout())
    lay.layout().addWidget(cmd)
    lay.layout().addWidget(ravenor)

    ui.mainWidget.addTab(lay, QIcon(raven.API().CWD + 'icons/app.png'),
                         'Ravenor')
Esempio n. 10
0
    def testMerge(self):
        cfg1 = Config()
        cfg1.load(makeStream("merge_1"))
        cfg2 = Config(makeStream("merge_2"))
        ConfigMerger().merge(cfg1, cfg2)
        merged = cfg1
        cfg1 = Config()
        cfg1.load(makeStream("merge_1"))
        for i in xrange(0, 5):
            key = 'value%d' % (i + 1,)
            self.failUnless(key in merged)
        self.assertEqual(len(cfg1.value5) + len(cfg2.value5),
           len(merged.value5))
        cfg3 = Config()
        cfg3.load(makeStream("merge_3"))
        cfg4 = Config(makeStream("merge_4"))
        merger = ConfigMerger()
        self.assertRaises(ConfigError, merger.merge, cfg3, cfg4)

        cfg3 = Config(makeStream("merge_3"))
        cfg4 = Config(makeStream("merge_4"))
        merger = ConfigMerger(config.overwriteMergeResolve)
        merger.merge(cfg3, cfg4)
        self.assertEqual(False, cfg3['value1'])
        self.assertEqual(4, cfg3['value2'])

        def customMergeResolve(map1, map2, key):
            if key == "value3":
                rv = "overwrite"
            else:
                rv = config.overwriteMergeResolve(map1, map2, key)
            return rv

        cfg3 = Config(makeStream("merge_3"))
        cfg4 = Config(makeStream("merge_4"))
        merger = ConfigMerger(customMergeResolve)
        merger.merge(cfg3, cfg4)
        self.assertEqual("[2, 4, 6]", str(cfg3.value3))
        self.assertEqual("[1, 3, 5, 2, 4, 6]", str(cfg3.value4))
Esempio n. 11
0
    def testMerge(self):
        cfg1 = Config()
        cfg1.load(makeStream("merge_1"))
        cfg2 = Config(makeStream("merge_2"))
        ConfigMerger().merge(cfg1, cfg2)
        merged = cfg1
        cfg1 = Config()
        cfg1.load(makeStream("merge_1"))
        for i in range(0, 5):
            key = 'value%d' % (i + 1,)
            self.assertTrue(key in merged)
        self.assertEqual(len(cfg1.value5) + len(cfg2.value5),
           len(merged.value5))
        cfg3 = Config()
        cfg3.load(makeStream("merge_3"))
        cfg4 = Config(makeStream("merge_4"))
        merger = ConfigMerger()
        self.assertRaises(ConfigError, merger.merge, cfg3, cfg4)

        cfg3 = Config(makeStream("merge_3"))
        cfg4 = Config(makeStream("merge_4"))
        merger = ConfigMerger(config.overwriteMergeResolve)
        merger.merge(cfg3, cfg4)
        self.assertEqual(False, cfg3['value1'])
        self.assertEqual(4, cfg3['value2'])

        def customMergeResolve(map1, map2, key):
            if key == "value3":
                rv = "overwrite"
            else:
                rv = config.overwriteMergeResolve(map1, map2, key)
            return rv

        cfg3 = Config(makeStream("merge_3"))
        cfg4 = Config(makeStream("merge_4"))
        merger = ConfigMerger(customMergeResolve)
        merger.merge(cfg3, cfg4)
        self.assertEqual("[2, 4, 6]", str(cfg3.value3))
        self.assertEqual("[1, 3, 5, 2, 4, 6]", str(cfg3.value4))
def initialize(pattern = "^env.*\.cfg$"):
    global _CONFIG
    _merger = ConfigMerger(overwriteMergeResolve)
    conf_list = []
    m = re.compile(pattern)

    for path in sys.path:
        for root, directories, files in os.walk(path):
            for file_name in files:
                if not m.match(file_name):
                    continue
                
                conf_list.append(os.path.join(root, file_name))
    
    for file_path in conf_list:
        f = file(file_path)
        
        try:
            cfg = Config(f)
            _merger.merge(_CONFIG, cfg)
        except:
            _logger.warn('load config file %s failed.', file_path, exc_info = True)
        
    _logger.debug('all configurations: %s', _CONFIG)
Esempio n. 13
0
    def main(self):
        self.log.info('Start')
        self.log.debug('Started: ' + os.path.abspath(__file__))
        self.log.debug('Setting SIGTERM, SIGINT handlers')
        signal.signal(signal.SIGTERM, self.exit_handler)
        signal.signal(signal.SIGINT, self.exit_handler)

        # Read cam configs
        cam_cfg_dir = os.path.join(self.cfg_dir, self.cfg['cam_cfg_mask'])
        self.log.debug('Configs search path: ' + cam_cfg_dir)

        cam_cfg_list = glob2.glob(
            os.path.join(self.cfg_dir, self.cfg['cam_cfg_mask']))
        cam_cfg_list.remove(self.cfg_file)
        self.log.debug('Found configs: %s' % cam_cfg_list)

        if len(cam_cfg_list) == 0:
            self.log.critical('No cam config found. Exit')
            sys.exit(0)

        for cur_cam_cfg in cam_cfg_list:
            self.log.debug('Read cam config: ' + cur_cam_cfg)
            tmp_cfg = Config(open(cur_cam_cfg))
            cur_cam_cfg_active_flag = True

            try:
                tmp_cfg['active']
            except AttributeError:
                self.log.debug('active flag not found')
            else:
                cur_cam_cfg_active_flag = tmp_cfg['active']

            if cur_cam_cfg_active_flag:
                self.cam_cfg.append(tmp_cfg)
                self.cam_cfg_resolver_dict.clear()
                merger = ConfigMerger(resolver=self.configs_resolver)
                merger.merge(self.cam_cfg[-1], self.cfg)

                for key in self.cam_cfg_resolver_dict:
                    self.cam_cfg[-1][key] = self.cam_cfg_resolver_dict[key]

                self.log.debug('Loaded settings for: ' +
                               self.cam_cfg[-1]['name'])
            else:
                self.log.debug('Cam config is skipped due active flag: ' +
                               cur_cam_cfg)
        # End Read cam configs

        # Cleaner
        self.cfg['cleaner_max_removes_per_run'] = self.replacer(
            str(self.cfg['cleaner_max_removes_per_run']), 0)
        schedule.every(self.cfg['cleaner_run_every_minutes']).minutes.do(
            self.cleaner)
        # End Cleaner

        # PIDs full path
        for iterator, cam in enumerate(self.cam_cfg):
            try:
                pid_streamer = cam['pid_streamer']
            except AttributeError:
                self.log.debug('pid_streamer not found for cam: ' +
                               cam['name'])
                try:
                    pid_streamer = self.cfg['pid_streamer']
                except AttributeError:
                    self.log.critical("Can't find pid_streamer in config")
                    sys.exit(1)

            try:
                pid_capturer = cam['pid_capturer']
            except AttributeError:
                self.log.debug('pid_capturer not found for cam: ' +
                               cam['name'])
                try:
                    pid_capturer = self.cfg['pid_capturer']
                except AttributeError:
                    self.log.critical("Can't find pid_capturer in config")
                    sys.exit(1)

            self.cam_streamer_pid.append(
                self.replacer(os.path.join(self.cfg['pid_dir'], pid_streamer),
                              iterator))
            self.cam_capturer_pid.append(
                self.replacer(os.path.join(self.cfg['pid_dir'], pid_capturer),
                              iterator))
        # End PIDs full path

        self.kill_cams_process()
        self.write_main_pid()

        while self.main_loop_active_flag:
            for iterator, cam in enumerate(self.cam_cfg):
                if len(self.cam_streamer) == iterator:

                    # Create cam cap dir only if cap_cmd is not False
                    try:
                        cap_cmd = self.cam_cfg[iterator]['cap_cmd']
                    except AttributeError:
                        cap_cmd = None
                        self.log.debug('Capture command not found')

                    if cap_cmd is not False:
                        cap_dir_cam = self.replacer(self.cfg['cap_dir_cam'],
                                                    iterator)
                        if not os.path.exists(cap_dir_cam):
                            try:
                                os.makedirs(cap_dir_cam)
                            except OSError:
                                self.log.critical(
                                    'Failed to create directory: ' +
                                    cap_dir_cam)
                                sys.exit(1)
                    # End Create cam cap dir

                    self.cam_streamer_start_flag.append(True)

                    self.cam_streamer.append(None)
                    self.cam_streamer_start_time.append(0)
                    self.cam_streamer_poll_flag.append(False)

                    self.cam_capturer.append(None)
                    self.cam_capturer_start_flag.append(False)
                    self.cam_capturer_check_flag.append(False)
                else:
                    if self.cam_streamer[iterator].poll() is None:
                        self.log.debug('Streamer "%s" is alive' % cam['name'])
                    else:
                        self.log.warn('Streamer "%s" is dead (exit code: %s)' %
                                      (cam['name'],
                                       self.cam_streamer[iterator].returncode))
                        self.cam_streamer_start_flag[iterator] = True

                # Capturer alive check
                if self.cam_capturer_check_flag[iterator]:
                    if self.cam_capturer[iterator].poll() is None:
                        self.log.debug('Capturer "%s" is alive' % cam['name'])
                    else:
                        self.log.warn('Capturer "%s" is dead (exit code: %s)' %
                                      (cam['name'],
                                       self.cam_capturer[iterator].returncode))
                        self.cam_streamer_poll_flag[iterator] = True
                        self.cam_capturer_check_flag[iterator] = False
                # End Capturer alive check

                # Run streamer
                if self.cam_streamer_start_flag[iterator]:
                    self.log.info('Run "%s" streamer in background' %
                                  cam['name'])
                    self.cam_streamer[iterator] = self.bg_run(
                        cam['cmd'].strip(), self.cam_streamer_pid[iterator])
                    self.cam_streamer_start_time[iterator] = time.time()
                    self.cam_streamer_poll_flag[iterator] = True
                    self.cam_streamer_start_flag[iterator] = False
                # End Run streamer

                # Poll streamer
                if self.cam_streamer_poll_flag[iterator]:
                    cap_url = self.cfg['cap_url']
                    cap_url = self.replacer(cap_url, iterator)

                    self.log.debug('Getting HTTP status: ' + cap_url)
                    http_code = 0

                    try:
                        http_code = requests.head(cap_url,
                                                  timeout=1).status_code
                    except requests.exceptions.RequestException:
                        self.log.warn('Failed to connect: ' + cap_url)

                    if http_code != 0:
                        self.log.info('Checked "%s", status: %s' %
                                      (cam['name'], http_code))

                        if http_code == 200:
                            self.cam_capturer_start_flag[iterator] = True
                            self.cam_streamer_poll_flag[iterator] = False

                    start_time_delta = time.time(
                    ) - self.cam_streamer_start_time[iterator]
                    if self.cam_streamer_poll_flag[iterator]:
                        if start_time_delta > cam['max_start_seconds']:
                            self.log.warn('Time outed waiting data from: ' +
                                          cam['name'])
                            self.log.info('Kill: ' + cam['name'])
                            self.kill_cam_processes(iterator,
                                                    cam_reset_flag=True)
                            self.cam_streamer_start_flag[iterator] = True
                        else:
                            self.log.info('Attempt "%s": [%i/%i]' %
                                          (cam['name'], start_time_delta,
                                           cam['max_start_seconds']))
                # End Poll streamer

                # Run capturer
                if self.cam_capturer_start_flag[iterator]:
                    if self.cam_capturer[
                            iterator] is not None and self.cam_capturer[
                                iterator].poll() is None:
                        self.log.warn('Capturer "%s" is STILL alive' %
                                      cam['name'])
                    else:
                        cap_cmd = None
                        try:
                            cap_cmd = self.cam_cfg[iterator]['cap_cmd']
                        except AttributeError:
                            self.log.debug(
                                'Capture command not found in cam config. Using global'
                            )
                            try:
                                cap_cmd = self.cfg['cap_cmd']
                            except AttributeError:
                                self.log.critical(
                                    'Capture command not found. Exit')
                                self.exit_handler(None,
                                                  None,
                                                  log_signal=False,
                                                  exit_code=1)

                        if cap_cmd is not False:
                            cap_cmd = self.replacer(cap_cmd, iterator)

                            self.log.info('Run "%s" capturer in background' %
                                          cam['name'])
                            self.cam_capturer[iterator] = self.bg_run(
                                cap_cmd, self.cam_capturer_pid[iterator])
                            self.cam_capturer_check_flag[iterator] = True
                        else:
                            self.log.info('Capturer "%s" is turned off' %
                                          cam['name'])

                    self.cam_capturer_start_flag[iterator] = False
                # End Run capturer

            schedule.run_pending()
            time.sleep(1)

        self.log.info('Finish')