class ProducesKafkaMessages(DeviceMixinBase):
""" Device to produce messages to kafka. The method *send* can be used
to produce a timestamped message onto the topic. Kafka brokers
can be specified using the parameter *brokers*.
"""
parameters = {
'brokers':
Param('List of kafka hosts to be connected',
type=listof(host(defaultport=9092)),
mandatory=True,
preinit=True,
userparam=False),
'max_request_size':
Param('Maximum size of kafka message',
type=int,
default=16000000,
preinit=True,
userparam=False),
}
def doPreinit(self, mode):
if mode != SIMULATION:
self._producer = kafka.KafkaProducer(
bootstrap_servers=self.brokers,
max_request_size=self.max_request_size)
else:
self._producer = None
def doShutdown(self):
if self._producer:
self._producer.close()
def _setProducerConfig(self, **configs):
self.doShutdown()
self._producer = kafka.KafkaProducer(bootstrap_servers=self.brokers,
**configs)
def send(self, topic, message, key=None, timestamp=None, partition=None):
"""
Produces and flushes the provided message
:param topic: Topic on which the message is to be produced
:param message: Message
:param key: key, for a compacted topic
:param timestamp: message timestamp in milliseconds
:param partition: partition on which the message is to be produced
:return:
"""
self._producer.send(topic, message, key, partition, timestamp)
self._producer.flush()
class BaseCacheClient(Device):
"""
An extensible read/write client for the NICOS cache.
"""
parameters = {
'cache':
Param('"host[:port]" of the cache instance to connect to',
type=host(defaultport=DEFAULT_CACHE_PORT),
mandatory=True),
'prefix':
Param('Cache key prefix', type=str, mandatory=True),
}
remote_callbacks = True
_worker = None
_startup_done = None
def doInit(self, mode):
# Should the worker connect or disconnect?
self._should_connect = True
# this event is set as soon as:
# * the connection is established and the connect_action is done, or
# * the initial connection failed
# this prevents devices from polling parameter values before all values
# from the cache have been received
self._startup_done = threading.Event()
self._connected = False
self._socket = None
self._secsocket = None
self._sec_lock = threading.RLock()
self._prefix = self.prefix.strip('/')
if self._prefix:
self._prefix += '/'
self._selecttimeout = CYCLETIME # seconds
self._do_callbacks = self.remote_callbacks
self._disconnect_warnings = 0
# maps newprefix -> oldprefix without self._prefix prepended
self._inv_rewrites = {}
# maps oldprefix -> set of new prefixes without self._prefix prepended
self._rewrites = {}
self._prefixcallbacks = {}
self._stoprequest = False
self._queue = queue.Queue()
self._synced = True
# create worker thread, but do not start yet, leave that to subclasses
self._worker = createThread('CacheClient worker',
self._worker_thread,
start=False)
def _getCache(self):
return None
def doShutdown(self):
self._stoprequest = True
if self._worker and self._worker.is_alive():
self._worker.join()
def _connect(self):
self._do_callbacks = False
self._startup_done.clear()
self.log.debug('connecting to %s', self.cache)
try:
self._socket = tcpSocket(self.cache,
DEFAULT_CACHE_PORT,
timeout=5,
keepalive=10)
except Exception as err:
self._disconnect('unable to connect to %s: %s' % (self.cache, err))
else:
self.log.info('now connected to %s', self.cache)
self._connected = True
self._disconnect_warnings = 0
try:
self._connect_action()
except Exception as err:
self._disconnect('unable to init connection to %s: %s' %
(self.cache, err))
self._startup_done.set()
self._do_callbacks = self.remote_callbacks
def _disconnect(self, why=''):
self._connected = False
self._startup_done.clear()
if why:
if self._disconnect_warnings % 10 == 0:
self.log.warning(why)
self._disconnect_warnings += 1
if self._socket:
closeSocket(self._socket)
self._socket = None
# close secondary socket
with self._sec_lock:
if self._secsocket:
closeSocket(self._secsocket)
self._secsocket = None
self._disconnect_action()
def _wait_retry(self):
sleep(self._long_loop_delay)
def _wait_data(self):
pass
def _connect_action(self):
# send request for all keys and updates....
# (send a single request for a nonexisting key afterwards to
# determine the end of data)
msg = '@%s%s\n%s%s\n' % (self._prefix, OP_WILDCARD, END_MARKER, OP_ASK)
self._socket.sendall(to_utf8(msg))
# read response
data, n = b'', 0
sentinel = to_utf8(END_MARKER + OP_TELLOLD + '\n')
while not data.endswith(sentinel) and n < 1000:
data += self._socket.recv(BUFSIZE)
n += 1
# send request for all updates
msg = '@%s%s\n' % (self._prefix, OP_SUBSCRIBE)
self._socket.sendall(to_utf8(msg))
for prefix in self._prefixcallbacks:
msg = '@%s%s\n' % (prefix, OP_SUBSCRIBE)
self._socket.sendall(to_utf8(msg))
self._process_data(data)
def _disconnect_action(self):
pass
def _handle_msg(self, time, ttlop, ttl, tsop, key, op, value):
raise NotImplementedError('implement _handle_msg in subclasses')
def _process_data(self,
data,
sync_str=to_utf8(SYNC_MARKER + OP_TELLOLD),
lmatch=line_pattern.match,
mmatch=msg_pattern.match):
# n = 0
i = 0 # avoid making a string copy for every line
match = lmatch(data, i)
while match:
line = match.group(1)
i = match.end()
if sync_str in line:
self.log.debug('process data: received sync: %r', line)
self._synced = True
else:
msgmatch = mmatch(from_utf8(line))
# ignore invalid lines
if msgmatch:
# n += 1
try:
self._handle_msg(**msgmatch.groupdict())
except Exception:
self.log.exception('error handling message %r',
msgmatch.group())
# continue loop
match = lmatch(data, i)
# self.log.debug('processed %d items', n)
return data[i:]
def _worker_thread(self):
while True:
try:
self._worker_inner()
except Exception:
self.log.exception('exception in cache worker thread; '
'restarting (please report a bug)')
if self._stoprequest:
break # ensure we do not restart during shutdown
else:
# normal termination
break
def _worker_inner(self):
data = b''
process = self._process_data
while not self._stoprequest:
if self._should_connect:
if not self._socket:
self._connect()
if not self._socket:
self._wait_retry()
continue
else:
if self._socket:
self._disconnect()
self._wait_retry()
continue
# process data so far
data = process(data)
# wait for a whole line of data to arrive
while b'\n' not in data and self._socket and self._should_connect \
and not self._stoprequest:
# optionally do some action while waiting
self._wait_data()
if self._queue.empty():
# NOTE: the queue.empty() check is not 100% reliable, but
# that is not important here: all we care is about not
# having the select always return immediately for writing
writelist = []
else:
writelist = [self._socket]
# read or write some data
while 1:
try:
res = select.select([self._socket], writelist, [],
self._selecttimeout)
except EnvironmentError as e:
if e.errno == errno.EINTR:
continue
raise
except TypeError:
# socket was None, let the outer loop handle that
res = ([], [], [])
break
if res[1]:
# determine if something needs to be sent
tosend = ''
itemcount = 0
try:
# bunch a few messages together, but not unlimited
for _ in xrange(10):
tosend += self._queue.get(False)
itemcount += 1
except queue.Empty:
pass
# write data
try:
self._socket.sendall(to_utf8(tosend))
except Exception:
self._disconnect('disconnect: send failed')
# report data as processed, but then re-queue it to send
# after reconnect
for _ in range(itemcount):
self._queue.task_done()
data = b''
self._queue.put(tosend)
break
for _ in range(itemcount):
self._queue.task_done()
if res[0]:
# got some data
try:
newdata = self._socket.recv(BUFSIZE)
except Exception:
newdata = b''
if not newdata:
# no new data from blocking read -> abort
self._disconnect('disconnect: recv failed')
data = b''
break
data += newdata
if self._socket:
# send rest of data
tosend = ''
itemcount = 0
try:
while 1:
tosend += self._queue.get(False)
itemcount += 1
except queue.Empty:
pass
try:
self._socket.sendall(to_utf8(tosend))
except Exception:
self.log.debug('exception while sending last batch of updates',
exc=1)
# no reraise, we'll disconnect below anyways
for _ in range(itemcount):
self._queue.task_done()
# end of while loop
self._disconnect()
def _single_request(self, tosend, sentinel=b'\n', retry=2, sync=False):
"""Communicate over the secondary socket."""
if not self._socket:
self._disconnect('single request: no socket')
if not self._socket:
raise CacheError('cache not connected')
if sync:
# sync has to be false for lock requests, as these occur during startup
self._queue.join()
with self._sec_lock:
if not self._secsocket:
try:
self._secsocket = tcpSocket(self.cache, DEFAULT_CACHE_PORT)
except Exception as err:
self.log.warning(
'unable to connect secondary socket '
'to %s: %s', self.cache, err)
self._secsocket = None
self._disconnect('secondary socket: could not connect')
raise CacheError('secondary socket could not be created')
try:
# write request
# self.log.debug("get_explicit: sending %r", tosend)
self._secsocket.sendall(to_utf8(tosend))
# give 10 seconds time to get the whole reply
timeout = currenttime() + 10
# read response
data = b''
while not data.endswith(sentinel):
newdata = self._secsocket.recv(BUFSIZE) # blocking read
if not newdata:
raise socket.error('cache closed connection')
if currenttime() > timeout:
# do not just break, we need to reopen the socket
raise socket.error('getting response took too long')
data += newdata
except socket.error:
self.log.warning('error during cache query', exc=1)
closeSocket(self._secsocket)
self._secsocket = None
if retry:
for m in self._single_request(tosend, sentinel, retry - 1):
yield m
return
raise
lmatch = line_pattern.match
mmatch = msg_pattern.match
i = 0
# self.log.debug("get_explicit: data =%r", data)
match = lmatch(data, i)
while match:
line = match.group(1)
i = match.end()
msgmatch = mmatch(from_utf8(line))
if not msgmatch:
# ignore invalid lines
continue
# self.log.debug('line processed: %r', line)
yield msgmatch
match = lmatch(data, i)
def waitForStartup(self, timeout):
self._startup_done.wait(timeout)
def flush(self):
"""wait for empty output queue"""
self._synced = False
self._queue.put('%s%s\n' % (SYNC_MARKER, OP_ASK))
self._queue.join()
for _ in range(100):
# self.log.debug('flush; waiting for sync...')
if self._synced:
break
sleep(CYCLETIME)
def addPrefixCallback(self, prefix, function):
"""Add a "prefix" callback, which is called for every key and value
that does not match the prefix parameter of the client, but matches
the prefix given to this function.
"""
if prefix not in self._prefixcallbacks:
self._queue.put('@%s%s\n' % (prefix, OP_SUBSCRIBE))
self._prefixcallbacks[prefix] = function
def removePrefixCallback(self, prefix):
"""Remove a "prefix" callback.
This removes the callback previously installed by addPrefixCallback.
If prefix is unknown, then do nothing.
"""
if prefix in self._prefixcallbacks:
self._queue.put('@%s%s\n' % (prefix, OP_UNSUBSCRIBE))
del self._prefixcallbacks[prefix]
# methods to make this client usable as the main device in a simple session
def start(self, *args):
self._connect()
self._worker.start()
def wait(self):
while not self._stoprequest:
sleep(self._long_loop_delay)
if self._worker and self._worker.is_alive():
self._worker.join()
def quit(self, signum=None):
self.log.info('quitting on signal %s...', signum)
self._stoprequest = True
def lock(self, key, ttl=None, unlock=False, sessionid=None):
"""Locking/unlocking: opens a separate connection."""
tosend = '%s%s%s%s%s\n' % (self._prefix, key.lower(), OP_LOCK,
unlock and OP_LOCK_UNLOCK or OP_LOCK_LOCK,
sessionid or session.sessionid)
if ttl is not None:
tosend = ('+%s@' % ttl) + tosend
for msgmatch in self._single_request(tosend, sync=False):
if msgmatch.group('value'):
raise CacheLockError(msgmatch.group('value'))
return
# no response received; let's assume standalone mode
self.log.warning('allowing lock/unlock operation without cache '
'connection')
def unlock(self, key, sessionid=None):
return self.lock(key, ttl=None, unlock=True, sessionid=sessionid)
def storeSysInfo(self, service):
"""Store info about the service in the cache."""
if not self._socket:
return
try:
key, res = getSysInfo(service)
msg = '%s@%s%s%s\n' % (currenttime(), key, OP_TELL,
cache_dump(res))
self._socket.sendall(to_utf8(msg))
except Exception:
self.log.exception('storing sysinfo failed')
class PixelmanUDPChannel(ActiveChannel):
"""
Trigger detector controlled by Pixelman software via UDP service
One of the detectors at V20 can be triggered through a simple UDP
service which listens on the computer that controls the detector.
It expects a keyword to start acquisition and then sends a keyword
back once it's done. The service expects a new connection for each
data acquisition, so the connection is established in the doStart
method and removed in doFinish. In order to recover from
inconsistent states, the socket is also torn down in doStop,
although that won't stop the detector, just reset the connection.
At the moment it's not possible to obtain count information from
the service, but that may change in the future.
"""
parameters = {
'host':
Param('IP and port for Pixelman Detector UDP interface.', type=host()),
'acquire':
Param('Keyword to send for starting the acquisition', type=str),
'finished':
Param(
'Keyword to wait for to determine '
'whether the acquisition is done',
type=str),
'acquiring':
Param('Internal parameter to synchronise between processes.',
type=bool,
internal=True,
default=False,
mandatory=False,
settable=False)
}
parameter_overrides = {'ismaster': Override(default=True, settable=True)}
def valueInfo(self):
return Value(self.name,
unit=self.unit,
type='other',
fmtstr=self.fmtstr),
def doInit(self, mode):
self._socket = None
def doStart(self):
if self._socket is None:
self.log.debug('Socket is None, creating socket.')
pm_host, pm_port = self.host.split(':')
self.log.debug('Connection: ' + self.host)
self._socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self._socket.connect((pm_host, int(pm_port)))
self.log.debug('Sending Keyword: ' + self.acquire)
self._socket.sendall(self.acquire)
self._socket.setblocking(0)
self._setROParam('acquiring', True)
self.log.debug('Acquisition started')
else:
self.log.info(
'Socket already exists, starting again has no effect.')
def doFinish(self):
self.log.debug('Finishing...')
if self._socket is not None:
self.log.debug('Actually shutting down...')
self._socket.close()
self._socket = None
self._setROParam('acquiring', False)
def doRead(self, maxage=0):
return 0
def doStop(self):
self.doFinish()
def doStatus(self, maxage=0):
if not self._check_complete():
return status.BUSY, 'Acquiring...'
return status.OK, 'Idle'
def duringMeasureHook(self, elapsed):
return None
def _check_complete(self):
self.log.debug('Checking completion...')
if session.sessiontype != POLLER:
if self._socket is not None:
self.log.debug('Actually performing check...')
try:
data = self._socket.recv(1024)
self.log.debug('Got data: ' + data)
return data == self.finished
except socket.error:
return False
self.log.debug('Falling back to Cache...')
return not self.acquiring
class KafkaSubscriber(DeviceMixinBase):
""" Receives messages from Kafka, can subscribe to a topic and get all
new messages from the topic if required via a callback method
*new_message_callback*.
"""
parameters = {
'brokers': Param('List of kafka hosts to be connected',
type=listof(host(defaultport=9092)),
mandatory=True, preinit=True, userparam=False)
}
def doPreinit(self, mode):
if mode != SIMULATION:
self._consumer = kafka.KafkaConsumer(
bootstrap_servers=self.brokers,
auto_offset_reset='latest' # start at latest offset
)
else:
self._consumer = None
# Settings for thread to fetch new message
self._stoprequest = True
self._updater_thread = None
def doShutdown(self):
if self._updater_thread is not None:
self._stoprequest = True
if self._updater_thread.is_alive():
self._updater_thread.join()
if self._consumer:
self._consumer.close()
@property
def consumer(self):
return self._consumer
def subscribe(self, topic):
""" Create the thread that provides call backs on new messages
"""
# Remove all the assigned topics
self._consumer.unsubscribe()
topics = self._consumer.topics()
if topic not in topics:
raise ConfigurationError('Provided topic %s does not exist' % topic)
# Assign the partitions
partitions = self._consumer.partitions_for_topic(topic)
if not partitions:
raise ConfigurationError('Cannot query partitions for %s' % topic)
self._consumer.assign([kafka.TopicPartition(topic, p)
for p in partitions])
self._stoprequest = False
self._updater_thread = createThread('updater_' + topic,
self._get_new_messages)
self.log.debug('subscribed to updates from topic: %s' % topic)
def _get_new_messages(self):
while not self._stoprequest:
sleep(self._long_loop_delay)
messages = {}
data = self._consumer.poll(5)
for records in data.values():
for record in records:
messages[record.timestamp] = record.value
if messages:
self.new_messages_callback(messages)
else:
self.no_messages_callback()
self.log.debug("KafkaSubscriber thread finished")
def new_messages_callback(self, messages):
"""This method is called whenever a new messages appear on
the topic. The subclasses should define this method if
a callback is required when new messages appear.
:param messages: dict of timestamp and raw message
"""
def no_messages_callback(self):
"""This method is called if no messages are on the topic.
class KafkaSubscriber(DeviceMixinBase):
""" Receives messages from Kafka, can subscribe to a topic and get all
new messages from the topic if required via a callback method
*new_message_callback*.
"""
parameters = {
'brokers': Param('List of kafka hosts to be connected',
type=listof(host(defaultport=9092)),
default=['localhost'], preinit=True, userparam=False)
}
def doPreinit(self, mode):
self._consumer = kafka.KafkaConsumer(
bootstrap_servers=self.brokers,
auto_offset_reset='latest' # start at latest offset
)
# Settings for thread to fetch new message
self._stoprequest = True
self._updater_thread = None
def doShutdown(self):
if self._updater_thread is not None:
self._stoprequest = True
if self._updater_thread.is_alive():
self._updater_thread.join()
self._consumer.close()
@property
def consumer(self):
return self._consumer
def subscribe(self, topic):
""" Create the thread that provides call backs on new messages
"""
# Remove all the assigned topics
self._consumer.unsubscribe()
self._consumer.subscribe(topic)
# # Assign the partitions
# partitions = self._consumer.partitions_for_topic(topic)
# if not partitions:
# raise NicosError('Provided topic %s does not exist' % topic)
#
# self._consumer.assign([kafka.TopicPartition(topic, p)
# for p in partitions])
# self._stoprequest = False
# self._updater_thread = createThread('updater_' + topic,
# self._get_new_messages)
self.log.debug('subscribed to updates from topic: %s' % topic)
def _get_new_messages(self):
while not self._stoprequest:
sleep(self._long_loop_delay)
assignment = self._consumer.assignment()
messages = {}
end = self._consumer.end_offsets(list(assignment))
for p in assignment:
while self._consumer.position(p) < end[p]:
msg = next(self._consumer)
messages[msg.timestamp] = msg.value
self.new_messages_callback(messages)
def new_messages_callback(self, messages):
"""This method is called whenever a new messages appear on
the topic. The subclasses should define this method if
a callback is required when new messages appear.
:param messages: dict of timestamp and raw message
"""
pass
class NicosDaemon(Device):
"""
This class abstracts the main daemon process.
"""
attached_devices = {
'authenticators':
Attach(
'The authenticator devices to use for '
'validating users and passwords',
Authenticator,
multiple=True),
}
parameters = {
'server':
Param('Address to bind to (host or host[:port])',
type=host(defaultport=DEFAULT_PORT),
mandatory=True,
ext_desc='The default port is ``1301``.'),
'servercls':
Param('Server class used for creating transports '
'to each client',
type=str,
mandatory=False,
default='nicos.services.daemon.proto.classic.'
'Server'),
'serializercls':
Param(
'Serializer class used for serializing '
'messages transported from/to the server',
type=str,
mandatory=False,
default='nicos.protocols.daemon.classic.'
'ClassicSerializer'),
'maxlogins':
Param('Maximum number of simultaneous clients '
'served',
type=int,
default=10),
'updateinterval':
Param(
'Interval for watch expressions checking and'
' sending updates to the clients',
type=float,
unit='s',
default=0.2),
'trustedhosts':
Param('A list of trusted hosts allowed to log in',
type=listof(str),
ext_desc='An empty list means all hosts are '
'allowed.'),
'simmode':
Param('Whether to always start in dry run mode', type=bool),
'autosimulate':
Param('Whether to simulate scripts when running them',
type=bool,
default=False)
}
def doInit(self, mode):
# import server and serializer class
servercls = importString(self.servercls)
serialcls = importString(self.serializercls)
self._stoprequest = False
# the controller represents the internal script execution machinery
if self.autosimulate and not config.sandbox_simulation:
raise ConfigurationError('autosimulation configured but sandbox'
' deactivated')
self._controller = ExecutionController(self.log, self.emit_event,
'startup', self.simmode,
self.autosimulate)
# cache log messages emitted so far
self._messages = []
host, port = parseHostPort(self.server, DEFAULT_PORT)
# create server (transport + serializer)
self._server = servercls(self, (host, port), serialcls())
self._watch_worker = createThread('daemon watch monitor',
self._watch_entry)
def _watch_entry(self):
"""
This thread checks for watch value changes periodically and sends out
events on changes.
"""
# pre-fetch attributes for speed
ctlr, intv, emit, sleep = self._controller, self.updateinterval, \
self.emit_event, time.sleep
lastwatch = {}
while not self._stoprequest:
sleep(intv)
# new watch values?
watch = ctlr.eval_watch_expressions()
if watch != lastwatch:
emit('watch', watch)
lastwatch = watch
def emit_event(self, event, data, blobs=None):
"""Emit an event to all handlers."""
self._server.emit(event, data, blobs or [])
def emit_event_private(self, event, data, blobs=None):
"""Emit an event to only the calling handler."""
handler = self._controller.get_current_handler()
if handler:
self._server.emit(event, data, blobs or [], handler=handler)
def statusinfo(self):
self.log.info('got SIGUSR2 - current stacktraces for each thread:')
active = threading._active
for tid, frame in list(sys._current_frames().items()):
if tid in active:
name = active[tid].getName()
else:
name = str(tid)
self.log.info('%s: %s', name, formatExtendedStack(frame))
def start(self):
"""Start the daemon's server."""
self.log.info('NICOS daemon v%s started, starting server on %s',
nicos_version, self.server)
# startup the script thread
self._controller.start_script_thread()
self._worker = createThread('daemon server',
self._server.start,
args=(self._long_loop_delay, ))
def wait(self):
while not self._stoprequest:
time.sleep(self._long_loop_delay)
self._worker.join()
def quit(self, signum=None):
self.log.info('quitting on signal %s...', signum)
self._stoprequest = True
self._server.stop()
self._worker.join()
self._server.close()
def current_script(self):
return self._controller.current_script
def current_user(self):
return getattr(self._controller.thread_data, 'user', system_user)
def get_authenticators(self):
return self._attached_authenticators
class KafkaCacheDatabase(MemoryCacheDatabase):
""" Cache database that stores cache in Kafka topics without History.
Current key and value pairs are stored in Kafka using log compaction.
The `CacheEntry` values can be serialized using the attached device. This
is then used to encode and decode entry instances while producing to
the Kafka topic and while consuming from it.
The data is stored in the Kafka topics using log compaction. Log
compaction ensures that Kafka will always retain at least the last
known value for each message key within the log of data for a single
topic partition.
The keys for partitioning are the keys received from cache server and
the timestamp of the message is same as the time of the cache entry.
If the provided `topic` does not exist, the cache database will not
be able to proceed. Use the command line kafka tool to create the topic.
This database will connect to the kafka and zookeeper services so
they should be running in background on the provided `hosts`. For
a basic getting started with kafka one can use the following guide:
https://kafka.apache.org/quickstart
Note that the minimum kafka broker version required for this
database to run properly is 0.11.0. Before this version timestamps
for messages could not be set.
The default behavior of Kafka does not ensure that the topic will
be cleaned up using the log compaction policy. For this to happen
one of the following two is to be done:
* Create topic by CLI Kafka tools using --config cleanup.policy=compact
* Set in server.properties log.cleanup.policy=compact
History is NOT supported with this database.
"""
parameters = {
'currenttopic':
Param('Kafka topic where the current values of cache are streamed',
type=str,
mandatory=True),
'brokers':
Param('List of Kafka bootstrap servers.',
type=listof(host(defaultport=9092)),
default=['localhost']),
}
attached_devices = {
'serializer':
Attach('Device to serialize the cache entry values',
CacheEntrySerializer,
optional=False)
}
def doInit(self, mode):
MemoryCacheDatabase.doInit(self, mode)
# Create the producer
self._producer = KafkaProducer(bootstrap_servers=self.brokers)
# Create the consumer
self._consumer = KafkaConsumer(
bootstrap_servers=self.brokers,
auto_offset_reset='earliest' # start at earliest topic
)
# Give up if the topic does not exist
if self.currenttopic not in self._consumer.topics():
raise ConfigurationError(
'Topic "%s" does not exit. Create this topic and restart.' %
self.currenttopic)
# Assign the partitions
partitions = self._consumer.partitions_for_topic(self.currenttopic)
self._consumer.assign(
[TopicPartition(self.currenttopic, p) for p in partitions])
# Cleanup thread configuration
self._stoprequest = False
self._cleaner = createThread('cleaner', self._clean, start=False)
def doShutdown(self):
self._consumer.close()
self._producer.close()
# Stop the cleaner thread
self._stoprequest = True
self._cleaner.join()
def initDatabase(self):
self.log.info('Reading messages from kafka topic - %s',
self.currenttopic)
now = currenttime()
message_count = 0
end = self._consumer.end_offsets(list(self._consumer.assignment()))
for partition in self._consumer.assignment():
while self._consumer.position(partition) < end[partition]:
msg = next(self._consumer)
message_count += 1
if msg.value is not None:
_, entry = self._attached_serializer.decode(msg.value)
if entry is not None and entry.value is not None:
# self.log.debug('%s (%s): %s -> %s', msg.offset,
# msg.timestamp, msg.key, entry)
if entry.ttl and entry.time + entry.ttl < now:
entry.expired = True
self._db[msg.key] = [entry]
self._cleaner.start()
self.log.info('Processed %i messages.', message_count)
def _clean(self):
def cleanonce():
with self._db_lock:
for key, entries in iteritems(self._db):
entry = entries[-1]
if not entry.value or entry.expired:
continue
time = currenttime()
if entry.ttl and (entry.time + entry.ttl < time):
entry.expired = True
for client in self._server._connected.values():
client.update(key, OP_TELLOLD, entry.value, time,
None)
while not self._stoprequest:
sleep(self._long_loop_delay)
cleanonce()
def _update_topic(self, key, entry):
# This method is responsible to communicate and update all the
# topics that should be updated. Subclasses can (re)implement it
# if there are messages to be produced to other topics
self.log.debug('Writing: %s -> %s', key, entry.value)
# For the log-compacted topic key deletion happens when None is
# passed as the value for the key
value = None
if entry.value is not None:
# Only when the key deletion is not required
value = self._attached_serializer.encode(key, entry)
self._producer.send(topic=self.currenttopic,
value=value,
key=bytes(key),
timestamp_ms=int(entry.time * 1000))
# clear all local buffers and produce pending messages
self._producer.flush()
def tell(self, key, value, time, ttl, from_client):
if value is None:
# deletes cannot have a TTL
ttl = None
send_update = True
always_send_update = False
# remove no-store flag
if key.endswith(FLAG_NO_STORE):
key = key[:-len(FLAG_NO_STORE)]
always_send_update = True
try:
category, subkey = key.rsplit('/', 1)
except ValueError:
category = 'nocat'
subkey = key
newcats = [category]
if category in self._rewrites:
newcats.extend(self._rewrites[category])
for newcat in newcats:
key = newcat + '/' + subkey
with self._db_lock:
entries = self._db.setdefault(key, [])
if entries:
lastent = entries[-1]
if lastent.value == value and not lastent.expired:
# not a real update
send_update = False
thisent = CacheEntry(time, ttl, value)
entries[:] = [thisent]
if send_update:
self._update_topic(key, thisent)
if send_update or always_send_update:
for client in self._server._connected.values():
if client is not from_client and client.is_active():
client.update(key, OP_TELL, value or '', time, ttl)
class CacheServer(Device):
"""
The server class.
"""
parameters = {
'server':
Param('Address to bind to (host or host:port)',
type=host(defaultport=DEFAULT_CACHE_PORT),
mandatory=True,
ext_desc="The default port is ``14869``."),
}
attached_devices = {
'db': Attach('The cache database instance', CacheDatabase),
}
def doInit(self, mode):
self._stoprequest = False
# TCP server address if bound to TCP
self._boundto = None
# server sockets for TCP and UDP
self._serversocket = None
self._serversocket_udp = None
# worker connections
self._connected = {}
self._attached_db._server = self
self._connectionLock = threading.Lock()
def start(self, *startargs):
if config.instrument == 'demo' and 'clear' in startargs:
self._attached_db.clearDatabase()
self._attached_db.initDatabase()
self.storeSysInfo()
self._worker = createThread('server', self._server_thread)
def storeSysInfo(self):
key, res = getSysInfo('cache')
self._attached_db.tell(key, str(res), currenttime(), None, None)
def _bind_to(self, address, proto='tcp'):
# bind to the address with the given protocol; return socket and address
host, port = parseHostPort(address, DEFAULT_CACHE_PORT)
serversocket = socket.socket(
socket.AF_INET, proto == 'tcp' and socket.SOCK_STREAM
or socket.SOCK_DGRAM)
serversocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
if proto == 'udp':
# we want to be able to receive UDP broadcasts
serversocket.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
try:
serversocket.bind((socket.gethostbyname(host), port))
if proto == 'tcp':
serversocket.listen(50) # max waiting connections....
return serversocket, (host, port)
except Exception:
serversocket.close()
return None, None # failed, return None as indicator
def _server_thread(self):
self.log.info('server starting')
# bind UDP broadcast socket
self.log.debug('trying to bind to UDP broadcast')
self._serversocket_udp = self._bind_to('', 'udp')[0]
if self._serversocket_udp:
self.log.info('UDP bound to broadcast')
# now try to bind TCP socket, include 'MUST WORK' standalone names
self.log.debug('trying to bind to %s', self.server)
self._serversocket, self._boundto = self._bind_to(self.server)
# one of the must have worked, otherwise continuing makes no sense
if not self._serversocket and not self._serversocket_udp:
self._stoprequest = True
self.log.error("couldn't bind any sockets, giving up!")
return
if not self._boundto:
self.log.warning('starting main loop only bound to UDP broadcast')
else:
self.log.info('TCP bound to %s:%s', self._boundto[0],
self._boundto[1])
# now enter main serving loop
while not self._stoprequest:
# loop through connections, first to remove dead ones,
# secondly to try to reconnect
for addr, client in list(self._connected.items()):
if not client.is_active(): # dead or stopped
self.log.info('client connection %s closed', addr)
client.closedown()
client.join() # wait for threads to end
del self._connected[addr]
# now check for additional incoming connections
# build list of things to check
selectlist = []
if self._serversocket:
selectlist.append(self._serversocket)
if self._serversocket_udp:
selectlist.append(self._serversocket_udp)
# 3 times client-side timeout
res = select.select(selectlist, [], [], CYCLETIME * 3)
if not res[0]:
continue # nothing to read -> continue loop
# lock aginst code in self.quit
with self._connectionLock:
if self._stoprequest:
break
if self._serversocket in res[0]:
# TCP connection came in
conn, addr = self._serversocket.accept()
addr = 'tcp://%s:%d' % addr
self.log.info('new connection from %s', addr)
self._connected[addr] = CacheWorker(self._attached_db,
conn,
name=addr,
loglevel=self.loglevel)
elif self._serversocket_udp in res[0]:
# UDP data came in
data, addr = self._serversocket_udp.recvfrom(3072)
nice_addr = 'udp://%s:%d' % addr
self.log.info('new connection from %s', nice_addr)
self._connected[nice_addr] = CacheUDPWorker(
self._attached_db,
self._serversocket_udp,
name=nice_addr,
data=data,
remoteaddr=addr,
loglevel=self.loglevel)
if self._serversocket:
closeSocket(self._serversocket)
self._serversocket = None
def wait(self):
while not self._stoprequest:
sleep(self._long_loop_delay)
self._worker.join()
def quit(self, signum=None):
self.log.info('quitting on signal %s...', signum)
self._stoprequest = True
# without locking, the _connected list may not have all clients yet....
with self._connectionLock:
for client in list(self._connected.values()):
self.log.info('closing client %s', client)
if client.is_active():
client.closedown()
with self._connectionLock:
for client in list(self._connected.values()):
self.log.info('waiting for %s', client)
client.closedown() # make sure, the connection closes down
client.join()
self.log.info('waiting for server')
self._worker.join()
self.log.info('server finished')
class JustBinItDetector(Detector):
""" A "detector" that reads image data from just-bin-it.
Note: it only uses image channels.
"""
parameters = {
'brokers':
Param('List of kafka hosts to be connected',
type=listof(host(defaultport=9092)),
mandatory=True,
preinit=True,
userparam=False),
'command_topic':
Param(
'The topic to send just-bin-it commands to',
type=str,
userparam=False,
settable=False,
mandatory=True,
),
'response_topic':
Param(
'The topic where just-bin-it responses appear',
type=str,
userparam=False,
settable=False,
mandatory=True,
),
'ack_timeout':
Param(
'How long to wait for timeout on acknowledgement',
type=int,
default=5,
unit='s',
userparam=False,
settable=False,
),
}
parameter_overrides = {
'unit': Override(default='events', settable=False, mandatory=False),
'fmtstr': Override(default='%d'),
'liveinterval': Override(type=floatrange(0.5), default=1),
}
_last_live = 0
_presets = {}
_presetkeys = {'t'}
_ack_thread = None
_exit_thread = False
def doPreinit(self, mode):
self._command_sender = kafka.KafkaProducer(
bootstrap_servers=self.brokers)
# Set up the response message consumer
self._response_consumer = kafka.KafkaConsumer(
bootstrap_servers=self.brokers)
self._response_topic = kafka.TopicPartition(self.response_topic, 0)
self._response_consumer.assign([self._response_topic])
self._response_consumer.seek_to_end()
self.log.debug('Response topic consumer initial position = %s',
self._response_consumer.position(self._response_topic))
def doInit(self, _mode):
pass
def doPrepare(self):
for image_channel in self._attached_images:
image_channel.doPrepare()
def doStart(self):
self._last_live = -(self.liveinterval or 0)
# Generate a unique-ish id
unique_id = 'nicos-{}-{}'.format(self.name, int(time.time()))
self.log.debug('set unique id = %s', unique_id)
count_interval = self._presets.get('t', None)
config = self._create_config(count_interval, unique_id)
if count_interval:
self.log.info(
'Requesting just-bin-it to start counting for %s seconds',
count_interval)
else:
self.log.info('Requesting just-bin-it to start counting')
self._send_command(self.command_topic, json.dumps(config).encode())
# Tell the channels to start
for image_channel in self._attached_images:
image_channel.doStart()
# Check for acknowledgement of the command being received
self._exit_thread = False
self._ack_thread = createThread("jbi-ack", self._check_for_ack,
(unique_id, self.ack_timeout))
def _check_for_ack(self, identifier, timeout_duration):
timeout = int(time.time()) + timeout_duration
acknowledged = False
while not (acknowledged or self._exit_thread):
messages = self._response_consumer.poll(timeout_ms=50)
responses = messages.get(self._response_topic, [])
for records in responses:
msg = json.loads(records.value)
if 'msg_id' in msg and msg['msg_id'] == identifier:
acknowledged = self._handle_message(msg)
break
# Check for timeout
if not acknowledged and int(time.time()) > timeout:
err_msg = 'Count aborted as no acknowledgement received from ' \
'just-bin-it within timeout duration '\
f'({timeout_duration} seconds)'
self.log.error(err_msg)
break
if not acknowledged:
# Couldn't start histogramming, so stop the channels etc.
self._stop_histogramming()
for image_channel in self._attached_images:
image_channel.doStop()
def _handle_message(self, msg):
if 'response' in msg and msg['response'] == 'ACK':
self.log.info('Counting request acknowledged by just-bin-it')
return True
elif 'response' in msg and msg['response'] == 'ERR':
self.log.error('just-bin-it could not start counting: %s',
msg['message'])
else:
self.log.error(
'Unknown response message received from just-bin-it')
return False
def _send_command(self, topic, message):
self._command_sender.send(topic, message)
self._command_sender.flush()
def _create_config(self, interval, identifier):
histograms = []
for image_channel in self._attached_images:
histograms.append(image_channel.get_configuration())
config_base = {
'cmd': 'config',
'msg_id': identifier,
'histograms': histograms
}
if interval:
config_base['interval'] = interval
else:
# If no interval then start open-ended count
config_base['start'] = int(time.time()) * 1000
return config_base
def valueInfo(self):
return tuple(info for channel in self._attached_images
for info in channel.valueInfo())
def doRead(self, maxage=0):
return [
data for channel in self._attached_images
for data in channel.read(maxage)
]
def doReadArrays(self, quality):
return [image.doReadArray(quality) for image in self._attached_images]
def doFinish(self):
self._stop_ack_thread()
self._stop_histogramming()
def _stop_histogramming(self):
self._send_command(self.command_topic, b'{"cmd": "stop"}')
def doShutdown(self):
self._response_consumer.close()
def doSetPreset(self, **presets):
self._presets = presets
def doStop(self):
self._stop_ack_thread()
self._stop_histogramming()
def _stop_ack_thread(self):
if self._ack_thread and self._ack_thread.is_alive():
self._exit_thread = True
self._ack_thread.join()
self._ack_thread = None
def doStatus(self, maxage=0):
return multiStatus(self._attached_images, maxage)
def doReset(self):
pass
def doInfo(self):
return [
data for channel in self._attached_images
for data in channel.doInfo()
]
def duringMeasureHook(self, elapsed):
if elapsed > self._last_live + self.liveinterval:
self._last_live = elapsed
return LIVE
return None
def arrayInfo(self):
return tuple(image.arrayInfo() for image in self._attached_images)
class CacheKafkaForwarder(ForwarderBase, Device):
parameters = {
'brokers': Param('List of kafka hosts to be connected',
type=listof(host(defaultport=9092)),
mandatory=True, preinit=True, userparam=False
),
'output_topic': Param('The topic to send data to',
type=str, userparam=False, settable=False,
mandatory=True,
),
'dev_ignore': Param('Devices to ignore; if empty, all devices are '
'accepted', default=[],
type=listof(str),
),
'update_interval': Param('Time interval (in secs.) to send regular updates',
default=10.0, type=float, settable=False)
}
parameter_overrides = {
# Key filters are irrelevant for this collector
'keyfilters': Override(default=[], settable=False),
}
def doInit(self, mode):
self._dev_to_value_cache = {}
self._dev_to_status_cache = {}
self._dev_to_timestamp_cache = {}
self._producer = None
self._lock = Lock()
self._initFilters()
self._queue = queue.Queue(1000)
self._worker = createThread('cache_to_kafka', self._processQueue,
start=False)
self._regular_update_worker = createThread(
'send_regular_updates', self._poll_updates, start=False)
while not self._producer:
try:
self._producer = \
KafkaProducer(bootstrap_servers=self._config['brokers'])
except Exception as error:
self.log.error(
'Could not connect to Kafka - will try again soon: %s',
error)
time.sleep(5)
self.log.info('Connected to Kafka brokers %s', self._config['brokers'])
def _startWorker(self):
self._worker.start()
self._regular_update_worker.start()
def _poll_updates(self):
while True:
with self._lock:
for dev_name in set(self._dev_to_value_cache.keys()).union(
self._dev_to_status_cache.keys()):
if self._relevant_properties_available(dev_name):
self._push_to_queue(
dev_name,
self._dev_to_value_cache[dev_name],
self._dev_to_status_cache[dev_name],
self._dev_to_timestamp_cache[dev_name])
time.sleep(self.update_interval)
def _checkKey(self, key):
if key.endswith('/value') or key.endswith('/status'):
return True
return False
def _checkDevice(self, name):
if name not in self.dev_ignore:
return True
return False
def _putChange(self, timestamp, ttl, key, value):
if value is None:
return
dev_name = key[0:key.index('/')]
if not self._checkKey(key) or not self._checkDevice(dev_name):
return
self.log.debug('_putChange %s %s %s', key, value, time)
with self._lock:
if key.endswith('value'):
self._dev_to_value_cache[dev_name] = value
else:
self._dev_to_status_cache[dev_name] = convert_status(value)
timestamp_ns = int(float(timestamp) * 10 ** 9)
self._dev_to_timestamp_cache[dev_name] = timestamp_ns
# Don't send until have at least one reading for both value and status
if self._relevant_properties_available(dev_name):
self._push_to_queue(
dev_name,
self._dev_to_value_cache[dev_name],
self._dev_to_status_cache[dev_name],
timestamp_ns,)
def _relevant_properties_available(self, dev_name):
return dev_name in self._dev_to_value_cache \
and dev_name in self._dev_to_status_cache \
and dev_name in self._dev_to_timestamp_cache
def _push_to_queue(self, dev_name, value, status, timestamp):
try:
self._queue.put_nowait((dev_name, value, status, timestamp))
except queue.Full:
self.log.error('Queue full, so discarding older value(s)')
self._queue.get()
self._queue.put((dev_name, value, status, timestamp))
self._queue.task_done()
def _processQueue(self):
while True:
name, value, status, timestamp = self._queue.get()
try:
# Convert value from string to correct type
value = cache_load(value)
if not isinstance(value, str):
# Policy decision: don't send strings via f142
buffer = to_f142(name, value, status, timestamp)
self._send_to_kafka(buffer)
except Exception as error:
self.log.error('Could not forward data: %s', error)
self._queue.task_done()
def doShutdown(self):
self._producer.close()
def _send_to_kafka(self, buffer):
self._producer.send(self.output_topic, buffer)
self._producer.flush(timeout=3)
