def __init__(self, url=u'sqlite:////tmp/janitoo_db.sqlite', pkg_name='janitoo_db', ep_name='janitoo', conf_file=None, **kwargs):
"""
"""
self.pkg_name = pkg_name
self.ep_name = ep_name
if conf_file is None:
src = os.path.join(pkg_resources.resource_filename(pkg_resources.Requirement.parse(self.pkg_name), 'config'), u'alembic_template.conf')
else:
src = conf_file
options = JNTOptions({'conf_file':conf_file})
options.load()
alembic = options.get_options('database')
url = alembic['sqlalchemy.url']
file_ = os.path.join(tempfile.gettempdir(), u'jntal_%s.conf')%(''.join(random.SystemRandom().choice(string.ascii_uppercase + string.digits) for _ in range(26)))
shutil.copyfile(src, file_)
alConfig.__init__(self, file_=file_, ini_section='database', **kwargs)
config_path = pkg_resources.resource_filename(pkg_resources.Requirement.parse(self.pkg_name), 'config')
self.set_main_option("script_location", os.path.join(config_path, 'alembic'))
self.set_main_option("sqlalchemy.url", url)
version_locations = u"%s/alembic/versions %s/models/%s"%(config_path, config_path, self.ep_name)
for entrypoint in pkg_resources.iter_entry_points(group='janitoo.models'):
pkg = entrypoint.module_name.split('.')[0]
config_path = pkg_resources.resource_filename(pkg_resources.Requirement.parse(pkg), 'config')
version_locations += u" %s/models/%s"%(config_path, entrypoint.name)
self.set_main_option("version_locations", version_locations)
def test_100_node_state(self):
with mock.patch('sys.argv', [self.prog, 'start', '--conf_file=tests/data/test_nodeman.conf']):
options = vars(jnt_parse_args())
options = JNTOptions(options)
section = 'fake'
thread_uuid = options.get_option(section, 'uuid')
if thread_uuid == None:
thread_uuid = muuid.uuid1()
options.set_option(section, 'uuid', "%s"%thread_uuid)
node_state = JNTNodeMan(options, section, thread_uuid)
print node_state.state
hadds = { 0 : HADD%(self.add_ctrl,0),
}
node_state.start()
i = 0
while node_state.state != 'ONLINE' and i<120:
i += 1
print node_state.state
time.sleep(1)
self.assertEqual(node_state.state, 'ONLINE')
node_state.stop()
i = 0
while node_state.state != 'OFFLINE' and i<120:
i += 1
print node_state.state
time.sleep(1)
self.assertEqual(node_state.state, 'OFFLINE')
node_state = None
def test_052_dbserver_auto_migrate(self):
options = JNTOptions({'conf_file':self.getDataFile(self.server_conf)})
options.load()
options.set_option('database','auto_migrate', True)
self.start()
self.assertHeartbeatNode()
self.stop()
def setUp(self):
JNTTDBServer.onlyDockerTest()
JNTTDBServer.setUp(self)
tmp_conf = self.cpTempFile(self.server_conf)
options = JNTOptions(options={'conf_file':tmp_conf})
options.set_option('database', 'sqlalchemy.url', self.dbconf[1]['dbconf'])
self.server_conf = tmp_conf
def test_100_node_state(self):
with mock.patch("sys.argv", [self.prog, "start", "--conf_file=tests/data/test_nodeman.conf"]):
options = vars(jnt_parse_args())
options = JNTOptions(options)
section = "fake"
thread_uuid = options.get_option(section, "uuid")
if thread_uuid == None:
thread_uuid = muuid.uuid1()
options.set_option(section, "uuid", "%s" % thread_uuid)
node_state = JNTNodeMan(options, section, thread_uuid)
print node_state.state
hadds = {0: HADD % (self.add_ctrl, 0)}
node_state.start()
i = 0
while node_state.state != "ONLINE" and i < 120:
i += 1
print node_state.state
time.sleep(1)
self.assertEqual(node_state.state, "ONLINE")
node_state.stop()
i = 0
while node_state.state != "OFFLINE" and i < 120:
i += 1
print node_state.state
time.sleep(1)
self.assertEqual(node_state.state, "OFFLINE")
node_state = None
class OptionsConfig(Config):
"""Load options from a common config file
"""
def __init__(self, conf_file):
"""Update Flask default data from janitoo option file
"""
Config.__init__(self)
if not os.path.isfile(conf_file):
raise RuntimeError("Can't find %s" %conf_file )
self.CONF_FILE = conf_file
self.options = JNTOptions({"conf_file":conf_file})
self.options.load()
if 'hostname' not in self.options.data or self.options.data['hostname'] is None:
self.options.data['hostname'] = socket.gethostname()
system = self.options.data
webapp = self.options.get_options('webapp')
database = self.options.get_options('database')
#~ try:
#~ flask = self.options.get_options('flask')
#~ except ConfigParser.NoSectionError:
#~ flask = {}
self.SQLALCHEMY_DATABASE_URI = database['sqlalchemy.url']
if 'host' in webapp and 'port' in webapp:
self.SERVER_NAME = "%s:%s"%(webapp['host'], webapp['port'])
if 'cache_dir' in system:
self.CACHE_DIR = os.path.join(system['cache_dir'], 'flask_cache_store')
def test_001_broadcast_secondary(self):
#~ self.wipTest("Freeze")
with mock.patch('sys.argv', [self.prog, 'start', '--conf_file=tests/data/test_nodeman.conf']):
options = vars(jnt_parse_args())
options = JNTOptions(options)
section = 'fake'
thread_uuid = options.get_option(section, 'uuid')
if thread_uuid == None:
thread_uuid = muuid.uuid1()
options.set_option(section, 'uuid', "%s"%thread_uuid)
self.nodeman = JNTBusNodeMan(options, FakeBus(options=options, product_name="Http server"), section, thread_uuid)
print self.nodeman.state
#~ hadds = { 0 : HADD%(self.node_add_ctrl,0),
#~ }
self.nodeman.start()
i = 0
while not self.nodeman.is_started and i<120:
i += 1
print self.nodeman.state
time.sleep(1)
self.assertEqual(self.nodeman.state, 'ONLINE')
with mock.patch('sys.argv', [self.prog, 'start', '--conf_file=tests/data/test_network.conf']):
options = vars(jnt_parse_args())
stopevent = threading.Event()
self.network = JNTNetwork(stopevent, JNTOptions(options), is_primary=False, is_secondary=True, do_heartbeat_dispatch=True)
print self.network.state
hadds = { 0 : HADD%(self.net_add_ctrl,0),
}
self.network.boot(hadds)
i = 0
while not self.network.is_started and i<150:
i += 1
print self.network.state
time.sleep(1)
print self.network.state
self.assertEqual(self.network.state, 'STARTED')
time.sleep(30)
print "network.nodes", self.network.nodes
print "network.users", self.network.users
print "network.configs", self.network.configs
print "network.basics", self.network.basics
print "network.systems", self.network.systems
print "network.commands", self.network.commands
print "HADD", HADD%(self.node_add_ctrl,0)
self.assertTrue(HADD%(self.node_add_ctrl,0) in self.network.nodes)
self.assertTrue(HADD%(self.node_add_ctrl,0) in self.network.systems)
self.assertTrue(HADD%(self.node_add_ctrl,0) in self.network.configs)
self.assertTrue(HADD%(self.node_add_ctrl,1) in self.network.nodes)
self.assertTrue(HADD%(self.node_add_ctrl,1) in self.network.systems)
self.assertTrue(HADD%(self.node_add_ctrl,1) in self.network.configs)
self.assertTrue(HADD%(self.node_add_ctrl,1) in self.network.basics)
self.assertTrue(HADD%(self.node_add_ctrl,1) in self.network.users)
self.assertTrue(HADD%(self.node_add_ctrl,1) in self.network.commands)
def test_001_engine(self):
options = JNTOptions({'conf_file':'tests/data/janitoo_db.conf'})
options.load()
engine = create_db_engine(options)
self.dbmaker = sessionmaker()
# Bind the sessionmaker to engine
self.dbmaker.configure(bind=engine)
self.dbsession = scoped_session(self.dbmaker)
Base.metadata.create_all(bind=engine)
def setUp(self):
JNTTServer.setUp(self)
options = JNTOptions({'conf_file':self.getDataFile(self.server_conf)})
options.load()
self.dbengine = create_db_engine(options)
self.dbmaker = sessionmaker()
# Bind the sessionmaker to engine
self.dbmaker.configure(bind=self.dbengine)
self.dbsession = scoped_session(self.dbmaker)
Base.metadata.drop_all(bind=self.dbengine)
def test_010_nodeman_sfm_states(self):
with mock.patch("sys.argv", [self.prog, "start", "--conf_file=tests/data/test_nodeman.conf"]):
options = vars(jnt_parse_args())
options = JNTOptions(options)
section = "fake"
thread_uuid = options.get_option(section, "uuid")
if thread_uuid == None:
thread_uuid = muuid.uuid1()
options.set_option(section, "uuid", "%s" % thread_uuid)
node_state = JNTNodeMan(options, section, thread_uuid, test=True)
node_state.start()
# ~ net_state.fsm_network_start()
while node_state.state != "ONLINE":
node_state.fsm_state_next()
node_state.stop()
def test_010_nodeman_sfm_states(self):
with mock.patch('sys.argv', [self.prog, 'start', '--conf_file=tests/data/test_nodeman.conf']):
options = vars(jnt_parse_args())
options = JNTOptions(options)
section = 'fake'
thread_uuid = options.get_option(section, 'uuid')
if thread_uuid == None:
thread_uuid = muuid.uuid1()
options.set_option(section, 'uuid', "%s"%thread_uuid)
node_state = JNTNodeMan(options, section, thread_uuid, test=True)
node_state.start()
#~ net_state.fsm_network_start()
while node_state.state != 'ONLINE':
node_state.fsm_state_next()
node_state.stop()
def __init__(self, options):
"""Init the server. Must be called at the begin of the children class.
"""
self._stopevent = threading.Event()
self.options = JNTOptions(options)
signal.signal(signal.SIGTERM, self.sigterm_handler)
#Need more tests
signal.signal(signal.SIGHUP, self.sighup_handler)
signal.signal(signal.SIGUSR1, self.sigusr1_handler)
self._threads = []
if 'conf_file' in self.options.data and self.options.data['conf_file'] is not None:
logging_fileConfig(self.options.data['conf_file'])
self.loop_sleep = 0.25
loop_sleep = self.options.get_option('system','loop_sleep')
if loop_sleep is not None:
try:
self.loop_sleep = int(loop_sleep)
except:
logger.exception("[%s] - Exception when retrieving value of loop_sleep. Use default value instead", self.__class__.__name__)
self.slow_start = 0.05
slow_start = self.options.get_option('system','slow_start')
if slow_start is not None:
try:
self.slow_start = int(slow_start)
except:
logger.exception("[%s] - Exception when retrieving value of slow_start. Use default value instead", self.__class__.__name__)
def assertNotInLogfile(self, expr="^ERROR "):
"""Assert an expression is not in logifle.
Must be called at the end of process, when the server has closed the logfile.
"""
self.assertTrue(self.server_conf is not None)
options = JNTOptions(options={"conf_file": self.getDataFile(self.server_conf)})
log_file_from_config = options.get_option("handler_file", "args", None)
self.assertTrue(log_file_from_config is not None)
# I know, it's bad
log_args = eval(log_file_from_config)
log_file_from_config = log_args[0]
self.assertFile(log_file_from_config)
found = False
with open(log_file_from_config, "r") as hand:
for line in hand:
print line
if re.search(expr, line):
found = True
self.assertFalse(found)
def test_301_busnode_find_helpers(self):
with mock.patch('sys.argv', [self.prog, 'start', '--conf_file=tests/data/test_bus.conf']):
options = vars(jnt_parse_args())
options = JNTOptions(options)
bus = FakeBus(options=options)
section = 'fake'
thread_uuid = options.get_option(section, 'uuid')
if thread_uuid == None:
thread_uuid = muuid.uuid1()
options.set_option(section, 'uuid', "%s"%thread_uuid)
node_state = JNTBusNodeMan(options, bus, section, thread_uuid)
print node_state.state
hadds = { 0 : HADD%(self.add_ctrl,0),
}
node_state.start()
i = 0
while node_state.state != 'ONLINE' and i<120:
i += 1
print node_state.state
time.sleep(1)
self.assertEqual(node_state.state, 'ONLINE')
try:
node = node_state.find_node('component1')
self.assertTrue('rrd1' in node.name)
node = node_state.find_node('componentbad')
self.assertEqual(node, None)
value = node_state.find_value('component1', 'heartbeat')
self.assertTrue('component1' in value.node_uuid)
self.assertEqual('heartbeat',value.uuid)
value = node_state.find_value('componentbad', 'heartbeat')
self.assertEqual(value, None)
value = node_state.find_value('component1', 'badbeat')
self.assertEqual(value, None)
finally:
node_state.stop()
i = 0
while node_state.state != 'OFFLINE' and i<120:
i += 1
print node_state.state
time.sleep(1)
self.assertEqual(node_state.state, 'OFFLINE')
def test_301_busnode_find_helpers(self):
with mock.patch("sys.argv", [self.prog, "start", "--conf_file=tests/data/test_bus.conf"]):
options = vars(jnt_parse_args())
options = JNTOptions(options)
bus = FakeBus(options=options)
section = "fake"
thread_uuid = options.get_option(section, "uuid")
if thread_uuid == None:
thread_uuid = muuid.uuid1()
options.set_option(section, "uuid", "%s" % thread_uuid)
node_state = JNTBusNodeMan(options, bus, section, thread_uuid)
print node_state.state
hadds = {0: HADD % (self.add_ctrl, 0)}
node_state.start()
i = 0
while node_state.state != "ONLINE" and i < 120:
i += 1
print node_state.state
time.sleep(1)
self.assertEqual(node_state.state, "ONLINE")
try:
node = node_state.find_node("component1")
self.assertTrue("rrd1" in node.name)
node = node_state.find_node("componentbad")
self.assertEqual(node, None)
value = node_state.find_value("component1", "heartbeat")
self.assertTrue("component1" in value.node_uuid)
self.assertEqual("heartbeat", value.uuid)
value = node_state.find_value("componentbad", "heartbeat")
self.assertEqual(value, None)
value = node_state.find_value("component1", "badbeat")
self.assertEqual(value, None)
finally:
node_state.stop()
i = 0
while node_state.state != "OFFLINE" and i < 120:
i += 1
print node_state.state
time.sleep(1)
self.assertEqual(node_state.state, "OFFLINE")
def test_051_dbserver_no_auto_migrate(self):
options = JNTOptions({'conf_file':self.getDataFile(self.server_conf)})
options.load()
options.set_option('database','auto_migrate', False)
try:
with self.assertRaises(JanitooException):
self.start()
self.assertHeartbeatNode()
self.stop()
finally:
options.set_option('database','auto_migrate', True)
def test_112_node_start_wait_random_stop_more(self):
#~ self.skipTest("Pass but freeze on Docker/CI. Surely a non stopped thread in the state machine")
self.onlyCITest()
with mock.patch('sys.argv', [self.prog, 'start', '--conf_file=tests/data/test_nodeman.conf']):
options = vars(jnt_parse_args())
options = JNTOptions(options)
section = 'fake'
thread_uuid = options.get_option(section, 'uuid')
if thread_uuid == None:
thread_uuid = muuid.uuid1()
options.set_option(section, 'uuid', "%s"%thread_uuid)
node_state = JNTNodeMan(options, section, thread_uuid)
print node_state.state
hadds = { 0 : HADD%(self.add_ctrl,0),
}
node_state.start()
time.sleep(6)
node_state.stop()
i = 0
while node_state.state != 'OFFLINE' and i<120:
i += 1
print node_state.state
time.sleep(1)
node_state.start()
time.sleep(15)
node_state.stop()
i = 0
while node_state.state != 'OFFLINE' and i<120:
i += 1
print node_state.state
time.sleep(1)
node_state.start()
time.sleep(32)
node_state.stop()
i = 0
while node_state.state != 'OFFLINE' and i<120:
i += 1
print node_state.state
time.sleep(1)
def test_111_node_start_wait_random_stop(self):
# ~ self.skipTest("Pass but freeze on Docker/CI. Surely a non stopped thread in the state machine")
self.onlyCITest()
with mock.patch("sys.argv", [self.prog, "start", "--conf_file=tests/data/test_nodeman.conf"]):
options = vars(jnt_parse_args())
options = JNTOptions(options)
section = "fake"
thread_uuid = options.get_option(section, "uuid")
if thread_uuid == None:
thread_uuid = muuid.uuid1()
options.set_option(section, "uuid", "%s" % thread_uuid)
node_state = JNTNodeMan(options, section, thread_uuid)
print node_state.state
hadds = {0: HADD % (self.add_ctrl, 0)}
node_state.start()
time.sleep(4)
node_state.stop()
i = 0
while node_state.state != "OFFLINE" and i < 120:
i += 1
print node_state.state
time.sleep(1)
node_state.start()
time.sleep(8)
node_state.stop()
i = 0
while node_state.state != "OFFLINE" and i < 120:
i += 1
print node_state.state
time.sleep(1)
node_state.start()
time.sleep(22)
node_state.stop()
i = 0
while node_state.state != "OFFLINE" and i < 120:
i += 1
print node_state.state
time.sleep(1)
def __init__(self, options):
"""Init the server. Must be called at the begin of the children class.
"""
self._stopevent = threading.Event()
self._reloadevent = threading.Event()
self.options = JNTOptions(options)
signal.signal(signal.SIGTERM, self.sigterm_handler)
#Need more tests
signal.signal(signal.SIGHUP, self.sighup_handler)
signal.signal(signal.SIGUSR1, self.sigusr1_handler)
self._threads = []
if 'conf_file' in self.options.data and self.options.data['conf_file'] is not None:
logging_fileConfig(self.options.data['conf_file'])
self.loop_sleep = 0.25
self.gc_delay = 0
self.slow_start = 0.05
def __init__(self, _app, options):
"""The constructor"""
#~ print "*"*25, "init the listener"
threading.Thread.__init__(self)
self._stopevent = threading.Event( )
self.app = _app
self.section="webapp"
self.mqttc = None
self.options = JNTOptions(options)
self.hadds = {}
self.network = None
self.create_network()
Controller.__init__(self, self.network)
self.loop_sleep = 0.25
loop_sleep = self.options.get_option('system','loop_sleep', self.loop_sleep)
if loop_sleep is not None:
self.loop_sleep = loop_sleep
else:
logger.debug("[%s] - Can't retrieve value of loop_sleep. Use default value instead (%s)", self.__class__.__name__, self.loop_sleep)
self.extend_from_entry_points('janitoo_flask')
def __init__(self, options=None):
"""Initialise the worker
:param options: The options used to start the worker.
:type clientid: str
"""
threading.Thread.__init__(self)
self._stopevent = threading.Event()
self._reloadevent = threading.Event()
self.config_timeout_timer = None
self.config_timeout_delay = 3
self.loop_sleep = 0.1
self.slow_start = 0.2
if not hasattr(self, "options"):
self.options = JNTOptions(options)
self.uuid = None
if not hasattr(self, "section"):
self.section = None
if self.section is None:
self.init_section()
class BaseThread(threading.Thread):
def __init__(self, options=None):
"""Initialise the worker
:param options: The options used to start the worker.
:type clientid: str
"""
threading.Thread.__init__(self)
self._stopevent = threading.Event()
self._reloadevent = threading.Event()
self.config_timeout_timer = None
self.config_timeout_delay = 3
self.loop_sleep = 0.1
self.slow_start = 0.2
if not hasattr(self, "options"):
self.options = JNTOptions(options)
self.uuid = None
if not hasattr(self, "section"):
self.section = None
if self.section is None:
self.init_section()
def __del__(self):
"""
"""
try:
self.stop()
except Exception:
pass
def init_section(self):
"""Init the section. Must be overloaded
"""
self.section = None
def config_timeout_callback(self):
"""Called when configuration is finished.
"""
logger.debug(
"[%s] - Configuration timeout occurs after %s seconds ... reload thread",
self.__class__.__name__,
self.config_timeout_delay,
)
self.config_timeout_timer = None
self._reloadevent.set()
def stop_trigger_reload(self, timeout=None):
"""Stop reload trigger.
"""
if self.config_timeout_timer is not None:
self.config_timeout_timer.cancel()
self.config_timeout_timer = None
def trigger_reload(self, timeout=None):
"""Trigger the config_timeout_callback to reload config.
"""
if timeout is not None:
try:
self.config_timeout_delay = int(timeout)
except ValueError:
logger.warning(
"[%s] - C'ant set timeout_delay to %s seconds in trigger_reload", self.__class__.__name__, timeout
)
self.stop_trigger_reload()
if self._stopevent.isSet():
return
self.config_timeout_timer = threading.Timer(self.config_timeout_delay, self.config_timeout_callback)
self.config_timeout_timer.start()
def stop(self):
"""Stop the thread
"""
logger.debug("[%s] - Stop the thread", self.__class__.__name__)
self._stopevent.set()
self.stop_trigger_reload()
def reload(self):
"""Stop the thread
"""
logger.debug("[%s] - reload the thread", self.__class__.__name__)
self._reloadevent.set()
def pre_loop(self):
"""Launch before entering the run loop. The node manager is available.
"""
try:
self.loop_sleep = float(self.options.get_option(self.section, "loop_sleep"))
except Exception:
logger.info(
"[%s] - Can't set loop_sleep from configuration file. Using default value %s",
self.__class__.__name__,
self.loop_sleep,
)
try:
self.slow_start = int(self.options.get_option("system", "slow_start"))
except Exception:
logger.info(
"[%s] - Can't set slow_start from configuration file. Using default value %s",
self.__class__.__name__,
self.slow_start,
)
def post_loop(self):
"""Launch after finishing the run loop. The node manager is still available.
"""
pass
def post_run(self):
"""Launch before exititng the run methode. You should dereference here.
"""
pass
def resource_filename(self, path="public"):
"""Needed to publish static files
"""
return resource_filename(Requirement.parse(self.get_package_name().split(".")[0]), path)
def get_package_name(self):
"""Return the name of the package. Needed to publish static files
**MUST** be copy paste in every extension that publish statics files
"""
return __package__
class JNTServer(object):
"""The Janitoo base Server
"""
def __init__(self, options):
"""Init the server. Must be called at the begin of the children class.
"""
self._stopevent = threading.Event()
self._reloadevent = threading.Event()
self.options = JNTOptions(options)
signal.signal(signal.SIGTERM, self.sigterm_handler)
#Need more tests
signal.signal(signal.SIGHUP, self.sighup_handler)
signal.signal(signal.SIGUSR1, self.sigusr1_handler)
self._threads = []
if 'conf_file' in self.options.data and self.options.data['conf_file'] is not None:
logging_fileConfig(self.options.data['conf_file'])
self.loop_sleep = 0.25
self.gc_delay = 0
self.slow_start = 0.05
def __del__(self):
"""
"""
try:
self.stop()
except:
pass
def start(self):
"""Start the server. Must be called at the end of the children class.
"""
logger.info("[%s] - Start the server", self.__class__.__name__)
self._stopevent.clear()
def pre_loop(self):
"""Before enterig the loop
"""
loop_sleep = self.options.get_option('system','loop_sleep')
if loop_sleep is not None:
try:
self.loop_sleep = int(loop_sleep)
except:
logger.info("[%s] - Can't set loop_sleep from configuration file. Using default valuse %s", self.__class__.__name__, self.loop_sleep)
gc_delay = self.options.get_option('system','gc_delay')
if gc_delay is not None:
try:
self.gc_delay = int(gc_delay)
except:
logger.info("[%s] - Can't set gc_delay from configuration file. Using default valuse %s", self.__class__.__name__, self.gc_delay)
if self.gc_delay>0:
self.gc_next_run = datetime.datetime.now() + datetime.timedelta(seconds=self.gc_delay)
slow_start = self.options.get_option('system','slow_start')
if slow_start is not None:
try:
self.slow_start = int(slow_start)
except:
logger.info("[%s] - Can't set slow_start from configuration file. Using default valuse %s", self.__class__.__name__, self.slow_start)
for entry in iter_entry_points(group='janitoo.threads', name=None):
th=None
try:
logger.debug("[%s] - Load entry-point : %s", self.__class__.__name__, entry)
mkth = entry.load()
try:
th = mkth(self.options.data)
except:
logger.exception("[%s] - Exception when loading thread from entry_point : %s", self.__class__.__name__, entry.name)
except:
logger.exception("[%s] - Exception when loading thread from entry_point : %s", self.__class__.__name__, entry.name)
if th is not None:
self._threads.append(th)
for th in self._threads:
th.start()
self._stopevent.wait(self.slow_start)
if len(self._threads) == 0:
logger.error("[%s] - Can't find a thread to launch in the config file", self.__class__.__name__)
raise JanitooException(message="Can't find a thread to launch in the config file")
logger.info("[%s] - Loaded thread(s) from entry_point : %s", self.__class__.__name__, self._threads)
def post_loop(self):
"""After the loop
"""
for th in self._threads:
th.stop()
for th in self._threads:
if th.is_alive():
th.join()
self._threads.remove(th)
self._threads = []
def run(self):
"""Run the loop
"""
logger.debug("[%s] - Entering the server loop", self.__class__.__name__)
while not self._stopevent.isSet():
self._reloadevent.clear()
self.pre_loop()
while not self._reloadevent.isSet() and not self._stopevent.isSet():
self._stopevent.wait(self.loop_sleep)
if self.gc_delay>0 and self.gc_next_run < datetime.datetime.now():
gc.collect()
self.gc_next_run = datetime.datetime.now() + datetime.timedelta(seconds=self.gc_delay)
self.post_loop()
logger.info("[%s] - Stop the server", self.__class__.__name__)
logger.debug("[%s] - Exiting the server loop", self.__class__.__name__)
self.post_loop()
def stop(self):
"""Stop the server. Must be called at begin if overloaded in the children class
"""
self._stopevent.set( )
def reload_threads(self):
"""Reload the threads
"""
logger.debug("[%s] - Reload threads", self.__class__.__name__)
for th in self._threads:
th.trigger_reload()
def find_thread(self, section):
"""Find a thread using its oid (section)
"""
ths = [ th for th in self._threads if hasattr(th,'section') and th.section == section ]
if len(ths) == 0:
return None
elif len(ths)>1:
logger.error("[%s] - Found multiple threads with same section %s : %s", self.__class__.__name__, section, ths)
return None
logger.debug("[%s] - Found thread with section %s : %s", self.__class__.__name__,section, ths[0] )
return ths[0]
def reload(self):
"""Reload the server
"""
logger.info("[%s] - Reload the server", self.__class__.__name__)
self._reloadevent.set( )
#~ self.stop()
#~ while len(self._threads)>0:
#~ self._stopevent.wait(self.loop_sleep*10)
#~ self._stopevent.wait(1.5)
#~ self.start()
def flush(self):
"""Flush the server's data to disk
"""
pass
def resource_filename(self, path='public'):
"""Needed to publish static files
"""
return resource_filename(Requirement.parse(self.get_package_name().split('.')[0]), path)
def get_package_name(self):
"""Return the name of the package. Needed to publish static files
**MUST** be copy paste in every extension that publish statics files
"""
return __package__
def _get_egg_path(self):
"""#DEPRECATED Return the egg path of the module. Must be redefined in server class.
"""
raise JanitooNotImplemented('_get_egg_path not implemnted')
def sigterm_handler(self, signal, frame):
"""Catch SIGTERM signal
"""
print('TERM signal received : %s' % (signal))
logger.warning('[%s] - TERM signal received : %s', self.__class__.__name__, signal)
self.stop()
#~ sys.exit(0)
def sighup_handler(self, signal, frame):
"""Catch SIGHUP signal
"""
print('HUP signal received : %s' % (signal))
logger.warning('[%s] - HUP signal received : %s', self.__class__.__name__, signal)
self.reload()
#~ sys.exit(0)
def sigusr1_handler(self, signal, frame):
"""Catch SIGUSR1 signal
The server must flush its data to disk
The mosquitto broker use it to persist its database to disk.
"""
print('USR1 signal received : %s' % (signal))
logger.warning('[%s] - USR1 signal received : %s', self.__class__.__name__, signal)
self.reload()
class ListenerThread(threading.Thread, Controller):
""" The listener Tread
"""
def __init__(self, _app, options):
"""The constructor"""
#~ print "*"*25, "init the listener"
threading.Thread.__init__(self)
self._stopevent = threading.Event( )
self.app = _app
self.section="webapp"
self.mqttc = None
self.options = JNTOptions(options)
self.hadds = {}
self.network = None
self.create_network()
Controller.__init__(self, self.network)
self.loop_sleep = 0.25
loop_sleep = self.options.get_option('system','loop_sleep', self.loop_sleep)
if loop_sleep is not None:
self.loop_sleep = loop_sleep
else:
logger.debug("[%s] - Can't retrieve value of loop_sleep. Use default value instead (%s)", self.__class__.__name__, self.loop_sleep)
self.extend_from_entry_points('janitoo_flask')
def __del__(self):
"""
"""
try:
self.stop()
except Exception:
logger.debug("[%s] - Catched exception", self.__class__.__name__)
def create_network(self):
"""Create the listener on first call
"""
self.network = NetworkFlask(self.app, self._stopevent, self.options, is_primary=False, is_secondary=True, do_heartbeat_dispatch=False)
def boot(self):
"""configure the HADD address
"""
#~ print("*"*25, "boot the listener")
default_hadd = HADD%(9998,0)
hadd = self.options.get_option('webapp','hadd', default_hadd)
if default_hadd is None:
logger.debug("[%s] - Can't retrieve value of hadd. Use default value instead (%s)", self.__class__.__name__, default_hadd)
self.hadds = { 0 : hadd,
}
#~ print "*"*25, "booting"
def run(self):
"""The running method
"""
#~ print "*"*25, "start the listener"
logger.info("Start listener")
self.boot()
self.network.boot(self.hadds)
Controller.start_controller(self, self.section, self.options, cmd_classes=[COMMAND_DHCPD], hadd=self.hadds[0], name="Webapp Server",
product_name="Webapp Server", product_type="Webapp Server")
self._stopevent.wait(1.0)
Controller.start_controller_timer(self)
while not self._stopevent.isSet():
self._stopevent.wait(self.loop_sleep)
if self.network is not None:
self.network.stop()
def stop(self):
"""Stop the tread
"""
#~ print("*"*25, "stop the listener")
Controller.stop_controller_timer(self)
Controller.stop_controller(self)
logger.info("Stop listener")
self._stopevent.set()
if self.network is not None:
#~ print("*"*25, "stop the network")
self.network.stop()
for i in range(100):
if self.network.is_stopped:
break
else:
self._stopevent.wait(0.1)
self.network = None
#~ print("*"*25, "network is stopped")
def extend_from_entry_points(self, group):
""""Extend the listener with methods found in entrypoints
"""
for entrypoint in iter_entry_points(group = '%s.listener'%group):
logger.info('Extend listener with %s', entrypoint.module_name )
extend = entrypoint.load()
extend( self )
class JNTServer(object):
"""The Janitoo base Server
"""
def __init__(self, options):
"""Init the server. Must be called at the begin of the children class.
"""
self._stopevent = threading.Event()
self.options = JNTOptions(options)
signal.signal(signal.SIGTERM, self.sigterm_handler)
#Need more tests
signal.signal(signal.SIGHUP, self.sighup_handler)
signal.signal(signal.SIGUSR1, self.sigusr1_handler)
self._threads = []
if 'conf_file' in self.options.data and self.options.data['conf_file'] is not None:
logging_fileConfig(self.options.data['conf_file'])
self.loop_sleep = 0.25
loop_sleep = self.options.get_option('system','loop_sleep')
if loop_sleep is not None:
try:
self.loop_sleep = int(loop_sleep)
except:
logger.exception("[%s] - Exception when retrieving value of loop_sleep. Use default value instead", self.__class__.__name__)
self.slow_start = 0.05
slow_start = self.options.get_option('system','slow_start')
if slow_start is not None:
try:
self.slow_start = int(slow_start)
except:
logger.exception("[%s] - Exception when retrieving value of slow_start. Use default value instead", self.__class__.__name__)
def start(self):
"""Start the server. Must be called at the end of the children class.
"""
logger.info("[%s] - Start the server", self.__class__.__name__)
self._stopevent.clear()
self.start_threads()
def start_threads(self):
"""Start the threads associated to this server.
"""
for entry in iter_entry_points(group='janitoo.threads', name=None):
th=None
try:
mkth = entry.load()
th = mkth(self.options.data)
except ImportError:
logger.exception("[%s] - Exception when loading thread from entry_point", self.__class__.__name__)
if th is not None:
self._threads.append(th)
for th in self._threads:
th.start()
self._stopevent.wait(self.slow_start)
logger.info("[%s] - Loaded thread(s) from entry_point : %s", self.__class__.__name__, self._threads)
if len(self._threads) == 0:
logger.error("[%s] - Can't find a thread to launch in the config file", self.__class__.__name__)
raise JanitooException(message="Can't find a thread to launch in the config file")
logger.info("[%s] - Loaded thread(s) from entry_point : %s", self.__class__.__name__, self._threads)
def pre_loop(self):
"""Before enterig the loop
"""
pass
def post_loop(self):
"""After the loop
"""
pass
def run(self):
"""Run the loop
"""
i = 0
self.pre_loop()
while not self._stopevent.isSet():
i += 1
self._stopevent.wait(self.loop_sleep)
self.post_loop()
def stop(self):
"""Stop the server. Must be called at begin if overloaded in the children class
"""
logger.info("[%s] - Stop the server", self.__class__.__name__)
self._stopevent.set( )
for th in self._threads:
th.stop()
for th in self._threads:
if th.is_alive():
th.join()
self._threads = []
def reload_threads(self):
"""Reload the threads
"""
logger.debug("[%s] - Reload threads", self.__class__.__name__)
for th in self._threads:
th.trigger_reload()
def reload(self):
"""Reload the server
"""
logger.info("[%s] - Reload the server", self.__class__.__name__)
self.stop()
while len(self._threads)>0:
self._stopevent.wait(self.loop_sleep*10)
time.sleep(1.0)
self.start()
def flush(self):
"""Flush the server's data to disk
"""
pass
def _get_egg_path(self):
"""Return the egg path of the module. Must be redefined in server class.
"""
raise JanitooNotImplemented('_get_egg_path not implemnted')
def sigterm_handler(self, signal, frame):
"""Catch SIGTERM signal
"""
print('TERM signal received : %s' % (signal))
logger.warning('TERM signal received : %s', signal)
self.stop()
sys.exit(0)
def sighup_handler(self, signal, frame):
"""Catch SIGHUP signal
"""
print('HUP signal received : %s' % (signal))
logger.warning('HUP signal received : %s', signal)
self.reload()
sys.exit(0)
def sigusr1_handler(self, signal, frame):
"""Catch SIGUSR1 signal
The server must flush its data to disk
The mosquitto broker use it to persist its database to disk.
"""
print('USR1 signal received : %s' % (signal))
logger.warning('USR1 signal received : %s', signal)
self.reload()
sys.exit(0)
#!/usr/bin/python
#From https://gist.github.com/svdgraaf/198e2c0cf4cf0a031c84
import pygraphviz as pgv
import threading
from janitoo.options import JNTOptions
from janitoo_solarpump.solarpump import SolarpumpBus
bus = SolarpumpBus(options=JNTOptions({}))
fsm = bus.create_fsm()
fsm.show_graph(fname='rst/images/fsm_bus.png', prog='dot')
def configure_extensions(app, config):
"""Configures the extensions."""
# Flask-WTF CSRF
csrf.init_app(app)
# Flask-SQLAlchemy
db.init_app(app)
# Flask-Migrate
#~ migrate.init_app(app, db, directory="config", filename="janitoo_manager.conf", section="database")
# Flask-Mail
mail.init_app(app)
# Flask-Debugtoolbar
debugtoolbar.init_app(app)
# Flask-Themes
themes.init_themes(app, app_identifier="janitoo_manager")
# Flask-Login
login_manager.login_view = app.config["LOGIN_VIEW"]
login_manager.refresh_view = app.config["REAUTH_VIEW"]
login_manager.anonymous_user = GuestMan
@login_manager.user_loader
def load_user(user_id):
"""Loads the user. Required by the `login` extension."""
#~ unread_count = db.session.query(db.func.count(Conversation.id)).\
#~ filter(Conversation.unread,
#~ Conversation.user_id == user_id).subquery()
#~ u = db.session.query(User, unread_count).filter(User.id == user_id).\
#~ first()
u = db.session.query(UserMan).filter(UserMan.id == user_id).\
first()
if u:
user_instance = u
#~ user_instance, user_instance.pm_unread = u
return user_instance
else:
return None
login_manager.init_app(app)
# Flask-BabelEx
babel.init_app(app=app, default_domain=JanitooDomain(app))
@babel.localeselector
def get_locale():
# if a user is logged in, use the locale from the user settings
if current_user.is_authenticated and current_user.language:
return current_user.language
# otherwise we will just fallback to the default language
return flask_config["DEFAULT_LANGUAGE"]
# SocketIO
socketio.init_app(app)
# janitoo_flask
options = JNTOptions({"conf_file":config.CONF_FILE})
options.load()
janitoo.init_app(app, socketio, options=options.data, db=db)
janitoo.extend_network('janitoo_manager')
janitoo.extend_listener('janitoo_manager')
