def initialize(load_test_conf=False):
global _CONFIG
_merger = ConfigMerger(overwriteMergeResolve)
conf_list = []
test_conf_list = []
for path in sys.path:
file_list = glob.glob(os.path.join(path, "env*.conf"))
for file_path in file_list:
if file_path.find("_test.conf") >= 0:
test_conf_list.append(file_path)
else:
conf_list.append(file_path)
if load_test_conf:
all_conf_list = conf_list + test_conf_list
else:
all_conf_list = conf_list
_logger.debug("load config files %s", all_conf_list)
for file_path in all_conf_list:
f = file(file_path)
try:
cfg = Config(f)
_merger.merge(_CONFIG, cfg)
except:
_logger.warn("load config file %s failed.", file_path, exc_info=True)
_logger.debug("all configurations: %s", _CONFIG)
def showRavenor(ui):
sys.path.append('../Ravenor')
import raven
from winterstone.base import WinterConfig
from config import ConfigMerger
raven.API().CWD+='../Ravenor/'
ravenconfig=WinterConfig(file(raven.API().CWD + 'config/main.cfg'))
darniconfig=WinterConfig(file(CWD + 'config/main.cfg'))
merger = ConfigMerger(resolve)
merger.merge(darniconfig.options.app, ravenconfig.options.app)
API().config=darniconfig
from etherstone.base import EtherIntegration
wi=EtherIntegration(ui,UI=False)
ravenor=raven.Panel(ui)
ravenor.tree=raven.Tree(ravenor,ravenor.api.CWD)
newpanel=raven.NewForm(ravenor)
ui.sideBar = WinterSideBar(ui)
ui.createSBAction('squares','Content',ravenor.tree.getWidget(),toolbar=True,keyseq=darniconfig.options.app.tree_shortcut)
ui.createSBAction('newPage','New',newpanel,toolbar=True,keyseq=darniconfig.options.app.new_shortcut)
ui.createSBAction('newPage','Edit',newpanel,toolbar=False,keyseq=darniconfig.options.app.new_shortcut)
cmd=raven.CommandLine(ravenor)
lay=QWidget()
lay.setLayout(QVBoxLayout())
lay.layout().addWidget(cmd)
lay.layout().addWidget(ravenor)
ui.mainWidget.addTab(lay,QIcon(raven.API().CWD+'icons/app.png'),'Ravenor')
def merge_params(default_params, override_params):
merger = ConfigMerger(resolver=overwriteMergeResolve)
params = Config()
if default_params is not None:
merger.merge(params, default_params)
if override_params is not None:
merger.merge(params, override_params)
return params
def load(self, config_name):
cfg = Config(file(os.path.join(self.BASE_PATH, "base.cfg")))
if not config_name.endswith(".cfg"):
config_name = config_name + ".cfg"
cfg1 = Config(file(config_name))
self.sections_to_merge = cfg.sections_to_merge
merger = ConfigMerger(self.merge_by_overwrite)
merger.merge(cfg, cfg1)
return cfg
def load(_reload=False):
global THE_CONF
if _reload or THE_CONF is None:
THE_CONF = Config()
THE_CONF.load(file(CONF_FILE_COMMON))
THE_CONF.load(file(CONF_FILE_ENV))
if CONF_FILE_LOCAL is not None:
local_conf = Config()
merger = ConfigMerger(_local_resolver)
local_conf.load(file(CONF_FILE_LOCAL))
merger.merge(THE_CONF, local_conf)
def test_config():
cfg = Config(file("test_base.cfg"))
cfg2 = Config(file("test_override.cfg"))
merger = ConfigMerger(mergeByOverwrite)
merger.merge(cfg, cfg2)
assert cfg.train.mid_layer_sizes[0] == 99, "midlayer_sizes[0] should take value from override (99)"
assert cfg.train.inner_code_length == 300, "inner_code_length should be as defined in base (300)"
assert cfg.val1 == 20, "val1 should be as defined in override (10)"
assert (
cfg.train.n_ins == 20
), "n_ins should very cleverly combine the formula from base with value from override to give 20"
def showRavenor(ui):
sys.path.append('../Ravenor')
import raven
from winterstone.base import WinterConfig
from config import ConfigMerger
raven.API().CWD += '../Ravenor/'
ravenconfig = WinterConfig(file(raven.API().CWD + 'config/main.cfg'))
darniconfig = WinterConfig(file(CWD + 'config/main.cfg'))
merger = ConfigMerger(resolve)
merger.merge(darniconfig.options.app, ravenconfig.options.app)
API().config = darniconfig
from etherstone.base import EtherIntegration
wi = EtherIntegration(ui, UI=False)
ravenor = raven.Panel(ui)
ravenor.tree = raven.Tree(ravenor, ravenor.api.CWD)
newpanel = raven.NewForm(ravenor)
ui.sideBar = WinterSideBar(ui)
ui.createSBAction('squares',
'Content',
ravenor.tree.getWidget(),
toolbar=True,
keyseq=darniconfig.options.app.tree_shortcut)
ui.createSBAction('newPage',
'New',
newpanel,
toolbar=True,
keyseq=darniconfig.options.app.new_shortcut)
ui.createSBAction('newPage',
'Edit',
newpanel,
toolbar=False,
keyseq=darniconfig.options.app.new_shortcut)
cmd = raven.CommandLine(ravenor)
lay = QWidget()
lay.setLayout(QVBoxLayout())
lay.layout().addWidget(cmd)
lay.layout().addWidget(ravenor)
ui.mainWidget.addTab(lay, QIcon(raven.API().CWD + 'icons/app.png'),
'Ravenor')
def merge_params(default_params, override_params):
merger = ConfigMerger(resolver=overwriteMergeResolve);
params = Config();
merger.merge(params, default_params);
merger.merge(params, override_params);
return params;
def merge_params(default_params, override_params):
merger = ConfigMerger(resolver=overwriteMergeResolve)
params = Config()
if default_params is not None:
merger.merge(params, default_params)
if override_params is not None:
merger.merge(params, override_params)
return params
def initialize(pattern = "^env.*\.cfg$"):
global _CONFIG
_merger = ConfigMerger(overwriteMergeResolve)
conf_list = []
m = re.compile(pattern)
for path in sys.path:
for root, directories, files in os.walk(path):
for file_name in files:
if not m.match(file_name):
continue
conf_list.append(os.path.join(root, file_name))
for file_path in conf_list:
f = file(file_path)
try:
cfg = Config(f)
_merger.merge(_CONFIG, cfg)
except:
_logger.warn('load config file %s failed.', file_path, exc_info = True)
_logger.debug('all configurations: %s', _CONFIG)
def testMerge(self):
cfg1 = Config()
cfg1.load(makeStream("merge_1"))
cfg2 = Config(makeStream("merge_2"))
ConfigMerger().merge(cfg1, cfg2)
merged = cfg1
cfg1 = Config()
cfg1.load(makeStream("merge_1"))
for i in xrange(0, 5):
key = 'value%d' % (i + 1,)
self.failUnless(key in merged)
self.assertEqual(len(cfg1.value5) + len(cfg2.value5),
len(merged.value5))
cfg3 = Config()
cfg3.load(makeStream("merge_3"))
cfg4 = Config(makeStream("merge_4"))
merger = ConfigMerger()
self.assertRaises(ConfigError, merger.merge, cfg3, cfg4)
cfg3 = Config(makeStream("merge_3"))
cfg4 = Config(makeStream("merge_4"))
merger = ConfigMerger(config.overwriteMergeResolve)
merger.merge(cfg3, cfg4)
self.assertEqual(False, cfg3['value1'])
self.assertEqual(4, cfg3['value2'])
def customMergeResolve(map1, map2, key):
if key == "value3":
rv = "overwrite"
else:
rv = config.overwriteMergeResolve(map1, map2, key)
return rv
cfg3 = Config(makeStream("merge_3"))
cfg4 = Config(makeStream("merge_4"))
merger = ConfigMerger(customMergeResolve)
merger.merge(cfg3, cfg4)
self.assertEqual("[2, 4, 6]", str(cfg3.value3))
self.assertEqual("[1, 3, 5, 2, 4, 6]", str(cfg3.value4))
def testMerge(self):
cfg1 = Config()
cfg1.load(makeStream("merge_1"))
cfg2 = Config(makeStream("merge_2"))
ConfigMerger().merge(cfg1, cfg2)
merged = cfg1
cfg1 = Config()
cfg1.load(makeStream("merge_1"))
for i in range(0, 5):
key = 'value%d' % (i + 1,)
self.assertTrue(key in merged)
self.assertEqual(len(cfg1.value5) + len(cfg2.value5),
len(merged.value5))
cfg3 = Config()
cfg3.load(makeStream("merge_3"))
cfg4 = Config(makeStream("merge_4"))
merger = ConfigMerger()
self.assertRaises(ConfigError, merger.merge, cfg3, cfg4)
cfg3 = Config(makeStream("merge_3"))
cfg4 = Config(makeStream("merge_4"))
merger = ConfigMerger(config.overwriteMergeResolve)
merger.merge(cfg3, cfg4)
self.assertEqual(False, cfg3['value1'])
self.assertEqual(4, cfg3['value2'])
def customMergeResolve(map1, map2, key):
if key == "value3":
rv = "overwrite"
else:
rv = config.overwriteMergeResolve(map1, map2, key)
return rv
cfg3 = Config(makeStream("merge_3"))
cfg4 = Config(makeStream("merge_4"))
merger = ConfigMerger(customMergeResolve)
merger.merge(cfg3, cfg4)
self.assertEqual("[2, 4, 6]", str(cfg3.value3))
self.assertEqual("[1, 3, 5, 2, 4, 6]", str(cfg3.value4))
def main(self):
self.log.info('Start')
self.log.debug('Started: ' + os.path.abspath(__file__))
self.log.debug('Setting SIGTERM, SIGINT handlers')
signal.signal(signal.SIGTERM, self.exit_handler)
signal.signal(signal.SIGINT, self.exit_handler)
# Read cam configs
cam_cfg_dir = os.path.join(self.cfg_dir, self.cfg['cam_cfg_mask'])
self.log.debug('Configs search path: ' + cam_cfg_dir)
cam_cfg_list = glob2.glob(
os.path.join(self.cfg_dir, self.cfg['cam_cfg_mask']))
cam_cfg_list.remove(self.cfg_file)
self.log.debug('Found configs: %s' % cam_cfg_list)
if len(cam_cfg_list) == 0:
self.log.critical('No cam config found. Exit')
sys.exit(0)
for cur_cam_cfg in cam_cfg_list:
self.log.debug('Read cam config: ' + cur_cam_cfg)
tmp_cfg = Config(open(cur_cam_cfg))
cur_cam_cfg_active_flag = True
try:
tmp_cfg['active']
except AttributeError:
self.log.debug('active flag not found')
else:
cur_cam_cfg_active_flag = tmp_cfg['active']
if cur_cam_cfg_active_flag:
self.cam_cfg.append(tmp_cfg)
self.cam_cfg_resolver_dict.clear()
merger = ConfigMerger(resolver=self.configs_resolver)
merger.merge(self.cam_cfg[-1], self.cfg)
for key in self.cam_cfg_resolver_dict:
self.cam_cfg[-1][key] = self.cam_cfg_resolver_dict[key]
self.log.debug('Loaded settings for: ' +
self.cam_cfg[-1]['name'])
else:
self.log.debug('Cam config is skipped due active flag: ' +
cur_cam_cfg)
# End Read cam configs
# Cleaner
self.cfg['cleaner_max_removes_per_run'] = self.replacer(
str(self.cfg['cleaner_max_removes_per_run']), 0)
schedule.every(self.cfg['cleaner_run_every_minutes']).minutes.do(
self.cleaner)
# End Cleaner
# PIDs full path
for iterator, cam in enumerate(self.cam_cfg):
try:
pid_streamer = cam['pid_streamer']
except AttributeError:
self.log.debug('pid_streamer not found for cam: ' +
cam['name'])
try:
pid_streamer = self.cfg['pid_streamer']
except AttributeError:
self.log.critical("Can't find pid_streamer in config")
sys.exit(1)
try:
pid_capturer = cam['pid_capturer']
except AttributeError:
self.log.debug('pid_capturer not found for cam: ' +
cam['name'])
try:
pid_capturer = self.cfg['pid_capturer']
except AttributeError:
self.log.critical("Can't find pid_capturer in config")
sys.exit(1)
self.cam_streamer_pid.append(
self.replacer(os.path.join(self.cfg['pid_dir'], pid_streamer),
iterator))
self.cam_capturer_pid.append(
self.replacer(os.path.join(self.cfg['pid_dir'], pid_capturer),
iterator))
# End PIDs full path
self.kill_cams_process()
self.write_main_pid()
while self.main_loop_active_flag:
for iterator, cam in enumerate(self.cam_cfg):
if len(self.cam_streamer) == iterator:
# Create cam cap dir only if cap_cmd is not False
try:
cap_cmd = self.cam_cfg[iterator]['cap_cmd']
except AttributeError:
cap_cmd = None
self.log.debug('Capture command not found')
if cap_cmd is not False:
cap_dir_cam = self.replacer(self.cfg['cap_dir_cam'],
iterator)
if not os.path.exists(cap_dir_cam):
try:
os.makedirs(cap_dir_cam)
except OSError:
self.log.critical(
'Failed to create directory: ' +
cap_dir_cam)
sys.exit(1)
# End Create cam cap dir
self.cam_streamer_start_flag.append(True)
self.cam_streamer.append(None)
self.cam_streamer_start_time.append(0)
self.cam_streamer_poll_flag.append(False)
self.cam_capturer.append(None)
self.cam_capturer_start_flag.append(False)
self.cam_capturer_check_flag.append(False)
else:
if self.cam_streamer[iterator].poll() is None:
self.log.debug('Streamer "%s" is alive' % cam['name'])
else:
self.log.warn('Streamer "%s" is dead (exit code: %s)' %
(cam['name'],
self.cam_streamer[iterator].returncode))
self.cam_streamer_start_flag[iterator] = True
# Capturer alive check
if self.cam_capturer_check_flag[iterator]:
if self.cam_capturer[iterator].poll() is None:
self.log.debug('Capturer "%s" is alive' % cam['name'])
else:
self.log.warn('Capturer "%s" is dead (exit code: %s)' %
(cam['name'],
self.cam_capturer[iterator].returncode))
self.cam_streamer_poll_flag[iterator] = True
self.cam_capturer_check_flag[iterator] = False
# End Capturer alive check
# Run streamer
if self.cam_streamer_start_flag[iterator]:
self.log.info('Run "%s" streamer in background' %
cam['name'])
self.cam_streamer[iterator] = self.bg_run(
cam['cmd'].strip(), self.cam_streamer_pid[iterator])
self.cam_streamer_start_time[iterator] = time.time()
self.cam_streamer_poll_flag[iterator] = True
self.cam_streamer_start_flag[iterator] = False
# End Run streamer
# Poll streamer
if self.cam_streamer_poll_flag[iterator]:
cap_url = self.cfg['cap_url']
cap_url = self.replacer(cap_url, iterator)
self.log.debug('Getting HTTP status: ' + cap_url)
http_code = 0
try:
http_code = requests.head(cap_url,
timeout=1).status_code
except requests.exceptions.RequestException:
self.log.warn('Failed to connect: ' + cap_url)
if http_code != 0:
self.log.info('Checked "%s", status: %s' %
(cam['name'], http_code))
if http_code == 200:
self.cam_capturer_start_flag[iterator] = True
self.cam_streamer_poll_flag[iterator] = False
start_time_delta = time.time(
) - self.cam_streamer_start_time[iterator]
if self.cam_streamer_poll_flag[iterator]:
if start_time_delta > cam['max_start_seconds']:
self.log.warn('Time outed waiting data from: ' +
cam['name'])
self.log.info('Kill: ' + cam['name'])
self.kill_cam_processes(iterator,
cam_reset_flag=True)
self.cam_streamer_start_flag[iterator] = True
else:
self.log.info('Attempt "%s": [%i/%i]' %
(cam['name'], start_time_delta,
cam['max_start_seconds']))
# End Poll streamer
# Run capturer
if self.cam_capturer_start_flag[iterator]:
if self.cam_capturer[
iterator] is not None and self.cam_capturer[
iterator].poll() is None:
self.log.warn('Capturer "%s" is STILL alive' %
cam['name'])
else:
cap_cmd = None
try:
cap_cmd = self.cam_cfg[iterator]['cap_cmd']
except AttributeError:
self.log.debug(
'Capture command not found in cam config. Using global'
)
try:
cap_cmd = self.cfg['cap_cmd']
except AttributeError:
self.log.critical(
'Capture command not found. Exit')
self.exit_handler(None,
None,
log_signal=False,
exit_code=1)
if cap_cmd is not False:
cap_cmd = self.replacer(cap_cmd, iterator)
self.log.info('Run "%s" capturer in background' %
cam['name'])
self.cam_capturer[iterator] = self.bg_run(
cap_cmd, self.cam_capturer_pid[iterator])
self.cam_capturer_check_flag[iterator] = True
else:
self.log.info('Capturer "%s" is turned off' %
cam['name'])
self.cam_capturer_start_flag[iterator] = False
# End Run capturer
schedule.run_pending()
time.sleep(1)
self.log.info('Finish')
