def looping_call(loop_function, loop_seconds):
loop_call = LoopingCall(loop_function)
loop_call.clock = self.core_reactor
loop_call.start(loop_seconds)
def add_timeout(interval, callback, *args, **kwargs):
"""Add a timeout callback as a task."""
time_out_task = LoopingCall(callback, *args, **kwargs)
time_out_task.start(interval / 1000, now=False)
class Main(object):
def __init__(self):
self.args = args = parse_args()
self.config = load_config(args)
self.logconfig = load_config(args, 'logconfig')
verbosity_adjust = (args.verbose or 0) - (args.quiet or 0)
self.log = setup_logging(self.logconfig,
args.instance_id,
verbosity_adjust=verbosity_adjust,
cache_on_use=True)
self.log.info('core-number-extractor', regex=args.container_name_regex)
self.voltha_version = self.get_version()
self.log.info('VOLTHA version is %s' % self.voltha_version)
# configurable variables from voltha.yml file
#self.configurable_vars = self.config.get('Constants', {})
if not args.no_banner:
print_banner(self.log)
# Create a unique instance id using the passed-in instance id and
# UTC timestamp
current_time = arrow.utcnow().timestamp
self.instance_id = self.args.instance_id + '_' + str(current_time)
# Every voltha instance is given a core_storage id where the
# instance data is stored
self.core_store_id = None
self.startup_components()
if not args.no_heartbeat:
self.start_heartbeat()
self.start_kafka_heartbeat(self.instance_id)
self.manhole = None
def get_version(self):
path = defs['version_file']
if not path.startswith('/'):
dir = os.path.dirname(os.path.abspath(__file__))
path = os.path.join(dir, path)
path = os.path.abspath(path)
version_file = open(path, 'r')
v = version_file.read()
# Use Version to validate the version string - exception will be raised
# if the version is invalid
Version(v)
version_file.close()
return v
def start(self):
self.start_reactor() # will not return except Keyboard interrupt
def stop(self):
pass
def get_args(self):
"""Allow access to command line args"""
return self.args
def get_config(self):
"""Allow access to content of config file"""
return self.config
@inlineCallbacks
def startup_components(self):
try:
self.log.info('starting-internal-components',
internal_host=self.args.internal_host_address,
external_host=self.args.external_host_address,
interface=self.args.interface,
consul=self.args.consul,
etcd=self.args.etcd)
registry.register('main', self)
if self.args.backend == 'consul':
yield registry.register(
'coordinator',
Coordinator(
internal_host_address=self.args.internal_host_address,
external_host_address=self.args.external_host_address,
rest_port=self.args.rest_port,
instance_id=self.instance_id,
config=self.config,
consul=self.args.consul,
container_name_regex=self.args.container_name_regex)
).start()
elif self.args.backend == 'etcd':
yield registry.register(
'coordinator',
CoordinatorEtcd(
internal_host_address=self.args.internal_host_address,
external_host_address=self.args.external_host_address,
rest_port=self.args.rest_port,
instance_id=self.instance_id,
config=self.config,
etcd=self.args.etcd,
container_name_regex=self.args.container_name_regex)
).start()
self.log.info('waiting-for-config-assignment')
# Wait until we get a config id before we proceed
self.core_store_id, store_prefix = \
yield registry('coordinator').get_core_store_id_and_prefix()
self.log.info('store-id', core_store_id=self.core_store_id)
# Update the logger to output the vcore id.
self.log = update_logging(instance_id=self.instance_id,
vcore_id=self.core_store_id)
yield registry.register('grpc_server',
VolthaGrpcServer(
self.args.grpc_port)).start()
yield registry.register(
'core',
VolthaCore(instance_id=self.instance_id,
core_store_id=self.core_store_id,
grpc_port=self.args.grpc_port,
version=self.voltha_version,
log_level=LogLevel.INFO)
).start(config_backend=load_backend(store_id=self.core_store_id,
store_prefix=store_prefix,
args=self.args))
init_rest_service(self.args.rest_port)
yield registry.register(
'kafka_proxy',
KafkaProxy(self.args.consul,
self.args.kafka,
config=self.config.get('kafka-proxy', {}))).start()
yield registry.register('frameio', FrameIOManager()).start()
yield registry.register(
'adapter_loader',
AdapterLoader(
config=self.config.get('adapter_loader', {}))).start()
yield registry.register(
'diag', Diagnostics(
config=self.config.get('diagnostics', {}))).start()
if self.args.manhole_port is not None:
self.start_manhole(self.args.manhole_port)
# Now that all components are loaded, in the scenario where this
# voltha instance is picking up an existing set of data (from a
# voltha instance that dies/stopped) then we need to setup this
# instance from where the previous one left
yield registry('core').reconcile_data()
# Now that the data is in memory and the reconcile process
# within the core has completed (the reconciliation may still be
# in progress with the adapters) we expose the NBI of voltha core
yield registry('core').register_grpc_service()
self.log.info('started-internal-services')
except Exception as e:
self.log.exception('Failure-to-start-all-components', e=e)
def start_manhole(self, port):
self.manhole = Manhole(port,
pws=dict(admin='adminpw'),
eventbus=EventBusClient(),
**registry.components)
@inlineCallbacks
def shutdown_components(self):
"""Execute before the reactor is shut down"""
self.log.info('exiting-on-keyboard-interrupt')
for component in reversed(registry.iterate()):
yield component.stop()
import threading
self.log.info('THREADS:')
main_thread = threading.current_thread()
for t in threading.enumerate():
if t is main_thread:
continue
if not t.isDaemon():
continue
self.log.info('joining thread {} {}'.format(
t.getName(), "daemon" if t.isDaemon() else "not-daemon"))
t.join()
def start_reactor(self):
from twisted.internet import reactor
reactor.callWhenRunning(
lambda: self.log.info('twisted-reactor-started'))
reactor.addSystemEventTrigger('before', 'shutdown',
self.shutdown_components)
reactor.run()
def start_heartbeat(self):
t0 = time.time()
t0s = time.ctime(t0)
def heartbeat():
self.log.debug(status='up', since=t0s, uptime=time.time() - t0)
lc = LoopingCall(heartbeat)
lc.start(10)
# Temporary function to send a heartbeat message to the external kafka
# broker
def start_kafka_heartbeat(self, instance_id):
# For heartbeat we will send a message to a specific "voltha-heartbeat"
# topic. The message is a protocol buf
# message
message = dict(type='heartbeat',
voltha_instance=instance_id,
ip=get_my_primary_local_ipv4())
topic = "voltha.heartbeat"
def send_msg():
try:
kafka_proxy = get_kafka_proxy()
if kafka_proxy and not kafka_proxy.is_faulty():
self.log.debug('kafka-proxy-available')
message['ts'] = arrow.utcnow().timestamp
self.log.debug('start-kafka-heartbeat')
kafka_proxy.send_message(topic, dumps(message))
else:
self.log.error('kafka-proxy-unavailable')
except Exception, e:
self.log.exception('failed-sending-message-heartbeat', e=e)
try:
lc = LoopingCall(send_msg)
lc.start(10)
except Exception, e:
self.log.exception('failed-kafka-heartbeat', e=e)
def on_map_change(self, map):
self.attacker = self.teams[ATTACKER_TEAM]
self.defender = self.attacker.other
self.defender_score_loop = LoopingCall(self.defender_score_cycle)
self.start_defender_score_loop()
protocol.on_map_change(self, map)
def __init__(self, incoming_box):
self.incoming_box = incoming_box
self.consumers = []
self._loop = LoopingCall(self._process)
def onJoin(self, details):
self.log.info("Session joined: {details}", details=details)
my_ip = get_ip_address()
joined_at = time.strftime("%H:%M")
# the voting subject we will display and vote for
subject = self.config.extra[u'subject']
# our quad, alphanumeric display: https://www.adafruit.com/products/2157
self._disp = QuadAlphanum(self.config.extra[u'i2c_address'])
self._disp.clear()
self._disp.setBrightness(
int(round(self.config.extra[u'brightness'] * 15)))
# display votes for subject on display
def setVotes(votes):
if votes < 10000:
text = "{:0>4d}".format(votes)
else:
text = "MANY"
self._disp.setMessage(text)
# get notified of new votes
def onVote(vote):
if vote[u'subject'] == subject:
setVotes(vote[u'votes'])
yield self.subscribe(onVote, u'io.crossbar.demo.vote.onvote')
# get notified of votes being reset
@inlineCallbacks
def onReset():
self._disp.setMessage('****')
yield sleep(.1)
setVotes(0)
yield self.subscribe(onReset, u'io.crossbar.demo.vote.onreset')
@inlineCallbacks
def displayNotice():
yield scrollText(
self._disp, "ip={} joined={} subject={} ...".format(
my_ip, joined_at, subject).upper())
# get the current votes
votes = yield self.call(u'io.crossbar.demo.vote.get')
for vote in votes:
if vote[u'subject'] == subject:
setVotes(vote[u'votes'])
# every couple of secs, display a notice
LoopingCall(displayNotice).start(60)
# init GPIO
GPIO.setwarnings(False)
GPIO.setmode(GPIO.BCM)
GPIO.cleanup()
GPIO.setup(self.config.extra[u'button_pin'], GPIO.IN)
self._button_state = False
@inlineCallbacks
def scan_buttons():
new_state = GPIO.input(self.config.extra[u'button_pin']) == 1
if new_state != self._button_state:
self.log.info("Button state change: {new_state}",
new_state=new_state)
if new_state:
yield self.call(u'io.crossbar.demo.vote.vote', subject)
self._button_state = new_state
# periodically scan buttons
scanner = LoopingCall(scan_buttons)
scanner.start(1. / 50.)
self.log.info("Votes listener ready!")
def __init__(self, protocol):
self.protocol = protocol
protocol.makeConnection(self)
self.input_loop = LoopingCall(self.get_input)
self.input_loop.start(0.01)
def crawl_spiders_in_loop():
configure_logging()
runner = CrawlerRunner()
task = LoopingCall(lambda: crawl_spiders(runner))
task.start(60 * 10)
reactor.run()
def scrape_loop_together():
configure_logging()
runner = CrawlerRunner()
task = LoopingCall(lambda: scrape_all_for_loop(runner))
task.start(60 * 20)
reactor.run()
def start(self):
lc = LoopingCall(self.scan_feed)
lc.start(self.interval)
lc2 = LoopingCall(self.scan_downloading)
lc2.start(self.interval * 2)
def start_collector(self, callback):
self.log.info("starting-pm-collection", device_name=self.name,
device_id=self.device.id)
prefix = 'voltha.{}.{}'.format(self.name, self.device.id)
self.lc = LoopingCall(callback, self.device.id, prefix)
self.lc.start(interval=self.default_freq / 10)
def startService(self):
self.loop = LoopingCall(self.poll)
base.ChangeSource.startService(self)
reactor.callLater(0, self.loop.start, self.pollInterval)
def __init__(self):
self.storage_reload_task = LoopingCall(reloadStorageSchemas)
self.aggregation_reload_task = LoopingCall(reloadAggregationSchemas)
def start_polling(self):
phone = gammu.StateMachine()
self.poller = LoopingCall(self.receive_and_send_messages, phone)
self.poller.start(self.poll_interval, True)
localConfig.access_token, localConfig.access_token_secret)
# betweezered_app
resource = WSGIResource(reactor, reactor.getThreadPool(), app)
site = Site(resource)
# run as script
if __name__ == '__main__':
# betweezered_app
logging.info('''
██████╗ ███████╗████████╗██╗ ██╗███████╗███████╗███████╗███████╗██████╗ ███████╗██████╗
██╔══██╗██╔════╝╚══██╔══╝██║ ██║██╔════╝██╔════╝╚══███╔╝██╔════╝██╔══██╗██╔════╝██╔══██╗
██████╔╝█████╗ ██║ ██║ █╗ ██║█████╗ █████╗ ███╔╝ █████╗ ██████╔╝█████╗ ██║ ██║
██╔══██╗██╔══╝ ██║ ██║███╗██║██╔══╝ ██╔══╝ ███╔╝ ██╔══╝ ██╔══██╗██╔══╝ ██║ ██║
██████╔╝███████╗ ██║ ╚███╔███╔╝███████╗███████╗███████╗███████╗██║ ██║███████╗██████╔╝
╚═════╝ ╚══════╝ ╚═╝ ╚══╝╚══╝ ╚══════╝╚══════╝╚══════╝╚══════╝╚═╝ ╚═╝╚══════╝╚═════╝
''')
# betweezered_app
reactor.listenTCP(localConfig.betweezered_app_port, site, interface="::")
logging.info("Listening on %s" % localConfig.betweezered_app_port)
# consume betweezered kafka topic
if localConfig.ts_kafka_consume == True:
lc = LoopingCall(bt_kafka.TwitterKafkaLooper().consume)
lc.start(localConfig.ts_kafka_loop_delay)
# fire reactor
reactor.run()
class Main(object):
def __init__(self):
self.args = args = parse_args()
self.config = load_config(args)
verbosity_adjust = (args.verbose or 0) - (args.quiet or 0)
self.log = setup_logging(self.config.get('logging', {}),
args.instance_id,
verbosity_adjust=verbosity_adjust)
self.log.info('container-number-extractor',
regex=args.container_name_regex)
self.adapter_version = self.get_version()
self.log.info('OpenONU-Adapter-Version', version=self.adapter_version)
if not args.no_banner:
print_banner(self.log)
self.adapter = None
# Create a unique instance id using the passed-in instance id and
# UTC timestamp
current_time = arrow.utcnow().timestamp
self.instance_id = self.args.instance_id + '_' + str(current_time)
self.core_topic = args.core_topic
self.listening_topic = args.name
self.startup_components()
if not args.no_heartbeat:
self.start_heartbeat()
self.start_kafka_cluster_heartbeat(self.instance_id)
def get_version(self):
path = defs['version_file']
if not path.startswith('/'):
dir = os.path.dirname(os.path.abspath(__file__))
path = os.path.join(dir, path)
path = os.path.abspath(path)
version_file = open(path, 'r')
v = version_file.read()
# Use Version to validate the version string - exception will be raised
# if the version is invalid
Version(v)
version_file.close()
return v
def start(self):
self.start_reactor() # will not return except Keyboard interrupt
def stop(self):
pass
def get_args(self):
"""Allow access to command line args"""
return self.args
def get_config(self):
"""Allow access to content of config file"""
return self.config
def _get_adapter_config(self):
cfg = AdapterConfig()
return cfg
@inlineCallbacks
def startup_components(self):
try:
self.log.info('starting-internal-components',
consul=self.args.consul,
etcd=self.args.etcd)
registry.register('main', self)
# Update the logger to output the vcore id.
self.log = update_logging(instance_id=self.instance_id,
vcore_id=None)
yield registry.register(
'kafka_cluster_proxy',
KafkaProxy(self.args.consul,
self.args.kafka_cluster,
config=self.config.get('kafka-cluster-proxy',
{}))).start()
config = self._get_adapter_config()
self.core_proxy = CoreProxy(
kafka_proxy=None,
default_core_topic=self.core_topic,
my_listening_topic=self.listening_topic)
self.adapter_proxy = AdapterProxy(
kafka_proxy=None,
core_topic=self.core_topic,
my_listening_topic=self.listening_topic)
self.adapter = BrcmOpenomciOnuAdapter(
core_proxy=self.core_proxy,
adapter_proxy=self.adapter_proxy,
config=config)
self.adapter.start()
openonu_request_handler = AdapterRequestFacade(
adapter=self.adapter, core_proxy=self.core_proxy)
yield registry.register(
'kafka_adapter_proxy',
IKafkaMessagingProxy(
kafka_host_port=self.args.kafka_adapter,
# TODO: Add KV Store object reference
kv_store=self.args.backend,
default_topic=self.args.name,
group_id_prefix=self.args.instance_id,
target_cls=openonu_request_handler)).start()
self.core_proxy.kafka_proxy = get_messaging_proxy()
self.adapter_proxy.kafka_proxy = get_messaging_proxy()
# retry for ever
res = yield self._register_with_core(-1)
self.log.info('started-internal-services')
except Exception as e:
self.log.exception('Failure-to-start-all-components', e=e)
@inlineCallbacks
def shutdown_components(self):
"""Execute before the reactor is shut down"""
self.log.info('exiting-on-keyboard-interrupt')
for component in reversed(registry.iterate()):
yield component.stop()
import threading
self.log.info('THREADS:')
main_thread = threading.current_thread()
for t in threading.enumerate():
if t is main_thread:
continue
if not t.isDaemon():
continue
self.log.info('joining thread {} {}'.format(
t.getName(), "daemon" if t.isDaemon() else "not-daemon"))
t.join()
def start_reactor(self):
from twisted.internet import reactor
reactor.callWhenRunning(
lambda: self.log.info('twisted-reactor-started'))
reactor.addSystemEventTrigger('before', 'shutdown',
self.shutdown_components)
reactor.run()
@inlineCallbacks
def _register_with_core(self, retries):
while 1:
try:
resp = yield self.core_proxy.register(
self.adapter.adapter_descriptor(),
self.adapter.device_types())
if resp:
self.log.info('registered-with-core',
coreId=resp.instance_id)
returnValue(resp)
except TimeOutError as e:
self.log.warn("timeout-when-registering-with-core", e=e)
if retries == 0:
self.log.exception("no-more-retries", e=e)
raise
else:
retries = retries if retries < 0 else retries - 1
yield asleep(defs['retry_interval'])
except Exception as e:
self.log.exception("failed-registration", e=e)
raise
def start_heartbeat(self):
t0 = time.time()
t0s = time.ctime(t0)
def heartbeat():
self.log.debug(status='up', since=t0s, uptime=time.time() - t0)
lc = LoopingCall(heartbeat)
lc.start(10)
# Temporary function to send a heartbeat message to the external kafka
# broker
def start_kafka_cluster_heartbeat(self, instance_id):
# For heartbeat we will send a message to a specific "voltha-heartbeat"
# topic. The message is a protocol buf
# message
message = dict(type='heartbeat',
adapter=self.args.name,
instance=instance_id,
ip=get_my_primary_local_ipv4())
topic = defs['heartbeat_topic']
def send_msg(start_time):
try:
kafka_cluster_proxy = get_kafka_proxy()
if kafka_cluster_proxy and not kafka_cluster_proxy.is_faulty():
# self.log.debug('kafka-proxy-available')
message['ts'] = arrow.utcnow().timestamp
message['uptime'] = time.time() - start_time
# self.log.debug('start-kafka-heartbeat')
kafka_cluster_proxy.send_message(topic, dumps(message))
else:
self.log.error('kafka-proxy-unavailable')
except Exception, e:
self.log.exception('failed-sending-message-heartbeat', e=e)
try:
t0 = time.time()
lc = LoopingCall(send_msg, t0)
lc.start(10)
except Exception, e:
self.log.exception('failed-kafka-heartbeat', e=e)
def run(self):
JobQueue.killall()
task = LoopingCall(self.check)
JobQueue.threads = [task]
task.start(self.interval)
print "Started polling jobs, every %d seconds." % (self.interval, )
def _start():
self.task = LoopingCall(self.call[0], *self.call[1],
**self.call[2])
self.task.clock = self.clock
self.deferred = self.task.start(ticks, now)
packet = json.dumps([{
"packet": ClientPackets.CHealthUpdate,
"chp": chp,
"mhp": mhp
}])
self.sendData(packet)
def sendMobHit(self, mob_id, damage):
packet = json.dumps([{
"packet": ClientPackets.CMobHit,
"id": mob_id,
"damage": damage
}])
self.sendData(packet)
if __name__ == '__main__':
global dataHandler
factory = gameClientFactory()
g.connector = reactor.connectTCP('127.0.0.1', 6000,
factory) # @UndefinedVariable
g.view = view.View()
g.tcpConn = TCPConnection(factory.protocol)
dataHandler = DataHandler()
#main loop here
lc = LoopingCall(g.view.tick) # @UndefinedVariable
d = lc.start(.03)
d.addErrback(log.err)
reactor.run() # @UndefinedVariable
def _test_tls_auth_denied(self):
"""
A MQTT client offering the wrong certificate won't be authenticated.
"""
reactor, router, server_factory, session_factory = build_mqtt_server()
real_reactor = selectreactor.SelectReactor()
logger = make_logger()
session, pump = connect_application_session(
server_factory, ObservingSession, component_config=ComponentConfig(realm="mqtt", controller=MockContainer()))
endpoint = create_listening_endpoint_from_config({
"type": "tcp",
"port": 1099,
"interface": "0.0.0.0",
"tls": {
"certificate": "server.crt",
"key": "server.key",
"dhparam": "dhparam",
"ca_certificates": [
"ca.cert.pem",
"intermediate.cert.pem"
]},
}, FilePath(__file__).sibling('certs').path, real_reactor, logger)
client_endpoint = create_connecting_endpoint_from_config({
"type": "tcp",
"host": "127.0.0.1",
"port": 1099,
"tls": {
# BAD key: trusted by the CA, but wrong ID
"certificate": "client_1.crt",
"hostname": "localhost",
"key": "client_1.key",
"ca_certificates": [
"ca.cert.pem",
"intermediate.cert.pem"
]},
}, FilePath(__file__).sibling('certs').path, real_reactor, logger)
p = []
l = endpoint.listen(server_factory)
class TestProtocol(Protocol):
data = b""
expected = (
ConnACK(session_present=False, return_code=1).serialise())
def dataReceived(self_, data):
self_.data = self_.data + data
if len(self_.data) == len(self_.expected):
self.assertEqual(self_.data, self_.expected)
real_reactor.stop()
@l.addCallback
def _listening(factory):
d = client_endpoint.connect(Factory.forProtocol(TestProtocol))
@d.addCallback
def _(proto):
p.append(proto)
proto.transport.write(
Connect(client_id="test123",
flags=ConnectFlags(clean_session=False)).serialise())
proto.transport.write(
Publish(duplicate=False, qos_level=1, retain=False, topic_name="test", payload=b"{}", packet_identifier=1).serialise())
lc = LoopingCall(pump.flush)
lc.clock = real_reactor
lc.start(0.01)
def timeout():
print("Timing out :(")
real_reactor.stop()
print(self.logs.log_text.getvalue())
# Timeout, just in case
real_reactor.callLater(10, timeout)
real_reactor.run()
client_protocol = p[0]
# We get a CONNECT
self.assertEqual(client_protocol.data,
ConnACK(session_present=False, return_code=1).serialise())
client_protocol.data = b""
pump.flush()
def initialize_system():
if hasattr(wallet, 'chain_broker'):
update = LoopingCall(app.update)
update.start(5)
def enable(self):
self.config = deluge.configmanager.ConfigManager("yarss.conf", DEFAULT_PREFS)
self.update_status_timer = LoopingCall(self.update_handler)
#self.update_handler()
self.update_status_timer.start(self.config['updatetime'])
def start(self):
lc = LoopingCall(self.scan_delete)
lc.start(self.interval)
def __init__(self):
self.rules = []
self.rules_file = None
self.read_task = LoopingCall(self.read_rules)
self.rules_last_read = 0.0
def process_words(self, data):
self.state.words = data[0]
self.state.guess = LoopingCall(self.make_guesses)
self.state.guess.start(self.step)
str(self.p2.rect.bottom) + "|" +
str(self.p1.rect.centerx) + "|" +
str(self.p1.rect.bottom) + "|" +
str(self.ball.rect.centerx) + "|" +
str(self.ball.rect.centery) + "|" + str(self.p1.points) +
"|" + str(self.p2.points))
elif self.menu.isMenu == True:
self.menu.tick()
if self.p2 != None and self.enters == 0:
tracker.player2.transport.write(
str(self.maxPts) + "|" + str(self.ceiling) + "|" +
str(self.walls))
# tell the clients the game ended
elif self.gameOver == True:
if self.p1.points >= self.maxPts:
tracker.player1.transport.write("win1")
tracker.player2.transport.write("win1")
elif self.p2.points >= self.maxPts:
tracker.player1.transport.write("win2")
tracker.player2.transport.write("win2")
tracker = Tracker()
gs = GameSpace()
lc = LoopingCall(gs.tick)
lc.start(1.0 / 30)
reactor.listenTCP(SERVER_PORT, ServerFactory())
reactor.run()
lc.stop()
def _init_(self, **kwargs):
self.calllater = {}
self.all_calls = reactor._delayedCalls # used by the web interface to list all unregistered call later's.
self.cleanup_expired_loop = LoopingCall(self.cleanup_expired)
self.cleanup_expired_loop.start(randint(120, 180), False)
def startService(self):
self._lc = LoopingCall(self._expire)
self._lc.clock = self._reactor
self._lc.start(1)
def __init__(self, protocol, call_on_exhaustion=None):
self.protocol = protocol
self.blocks = deque()
self.loop = LoopingCall(self.cycle)
self.loop.start(self.interval)
self.call_on_exhaustion = call_on_exhaustion
def __init__(self, core_reactor, options, config):
log_timer = Log(logging.DEBUG).start()
Log(logging.INFO).log("Service", "state", lambda: "[anode] initialising")
self.core_reactor = core_reactor
self.options = options
self.config = config
self.plugins = {}
self.certificate = pem.twisted.certificateOptionsFromFiles(self.options.certificate) \
if os.path.isfile(self.options.certificate) else None
self.web_ws = WebWsFactory(u"ws" + ("" if self.certificate is None else "s") + "://"
+ self.config["host"] + ":" + str(self.config["port"]), self, self.certificate)
self.web_ws.protocol = WebWs
self.web_rest = WebRest(self, "http" + ("" if self.certificate is None else "s") + "://"
+ self.config["host"] + ":" + str(self.config["port"]))
self.web_pool = HTTPConnectionPool(reactor, persistent=True)
self.publish_service = None
self.publish = "publish_host" in self.config and len(self.config["publish_host"]) > 0 and \
"publish_port" in self.config and self.config["publish_port"] > 0
if self.publish:
access_key = config["profile"]["VERNEMQ_ACCESS_KEY"] if "VERNEMQ_ACCESS_KEY" in config["profile"] else None
secret_key = config["profile"]["VERNEMQ_SECRET_KEY"] if "VERNEMQ_SECRET_KEY" in config["profile"] else None
mqtt_client_string = clientFromString(reactor, "tcp:" + self.config["publish_host"] + ":" + str(self.config["publish_port"]))
self.publish_service = MqttPublishService(mqtt_client_string, MQTTFactory(profile=MQTTFactory.PUBLISHER),
KEEPALIVE_DEFAULT_SECONDS, access_key, secret_key)
def looping_call(loop_function, loop_seconds):
loop_call = LoopingCall(loop_function)
loop_call.clock = self.core_reactor
loop_call.start(loop_seconds)
if "model_pull_seconds" in self.config and self.config["model_pull_seconds"] > 0:
model_pull = ModelPull(self, "pullmodel", {
"pool": self.web_pool, "db_dir": self.options.db_dir,
"profile": self.config["profile"],
"model_pull_region": self.config["model_pull_region"] if "model_pull_region" in self.config else S3_REGION,
"model_pull_bucket": (self.config["model_pull_bucket"] if "model_pull_bucket" in self.config else S3_BUCKET) + (
self.config["model_pull_bucket_snapshot"] if ("model_pull_bucket_snapshot" in self.config and
APP_VERSION.endswith("-SNAPSHOT")) else "")}, self.core_reactor)
looping_call(model_pull.poll, self.config["model_pull_seconds"])
if "plugin" in self.config and self.config["plugin"] is not None:
for plugin_name in self.config["plugin"]:
self.config["plugin"][plugin_name]["pool"] = self.web_pool
self.config["plugin"][plugin_name]["db_dir"] = self.options.db_dir
self.config["plugin"][plugin_name]["profile"] = self.config["profile"]
if self.publish_service is not None:
self.config["plugin"][plugin_name]["publish_service"] = self.publish_service
if "publish_batch_seconds" in self.config:
self.config["plugin"][plugin_name]["publish_batch_seconds"] = self.config["publish_batch_seconds"]
if "publish_status_topic" in self.config:
self.config["plugin"][plugin_name]["publish_status_topic"] = self.config["publish_status_topic"]
if "publish_push_data_topic" in self.config:
self.config["plugin"][plugin_name]["publish_push_data_topic"] = self.config["publish_push_data_topic"]
if "publish_push_metadata_topic" in self.config:
self.config["plugin"][plugin_name]["publish_push_metadata_topic"] = self.config["publish_push_metadata_topic"]
if "publish_batch_datum_topic" in self.config:
self.config["plugin"][plugin_name]["publish_batch_datum_topic"] = self.config["publish_batch_datum_topic"]
self.plugins[plugin_name] = Plugin.get(self, plugin_name, self.config["plugin"][plugin_name], self.core_reactor)
if "poll_seconds" in self.config["plugin"][plugin_name] and self.config["plugin"][plugin_name]["poll_seconds"] > 0:
looping_call(self.plugins[plugin_name].poll, self.config["plugin"][plugin_name]["poll_seconds"])
if "repeat_seconds" in self.config["plugin"][plugin_name] and self.config["plugin"][plugin_name]["repeat_seconds"] > 0:
looping_call(self.plugins[plugin_name].repeat, self.config["plugin"][plugin_name]["repeat_seconds"])
for plugin in self.plugins.itervalues():
if "history_partition_seconds" in self.config["plugin"][plugin.name] and \
self.config["plugin"][plugin.name]["history_partition_seconds"] > 0 and \
"repeat_seconds" in self.config["plugin"][plugin_name] and \
self.config["plugin"][plugin_name]["repeat_seconds"] >= 0:
time_current = plugin.get_time()
time_partition = self.config["plugin"][plugin.name]["history_partition_seconds"]
time_partition_next = time_partition - (time_current - plugin.get_time_period(time_current, time_partition))
plugin_partition_call = LoopingCall(self.plugins[plugin.name].repeat, force=True)
plugin_partition_call.clock = self.core_reactor
self.core_reactor.callLater(time_partition_next,
lambda _plugin_partition_call, _time_partition:
_plugin_partition_call.start(_time_partition), plugin_partition_call, time_partition)
if self.publish and "publish_batch_seconds" in self.config and self.config["publish_batch_seconds"] > 0:
looping_call(self.publish_datums, self.config["publish_batch_seconds"])
if "save_seconds" in self.config and self.config["save_seconds"] > 0:
looping_call(self.store_state, self.config["save_seconds"])
log_timer.log("Service", "timer", lambda: "[anode] initialised", context=self.__init__)
