class ProducesKafkaMessages(DeviceMixinBase): """ Device to produce messages to kafka. The method *send* can be used to produce a timestamped message onto the topic. Kafka brokers can be specified using the parameter *brokers*. """ parameters = { 'brokers': Param('List of kafka hosts to be connected', type=listof(host(defaultport=9092)), mandatory=True, preinit=True, userparam=False), 'max_request_size': Param('Maximum size of kafka message', type=int, default=16000000, preinit=True, userparam=False), } def doPreinit(self, mode): if mode != SIMULATION: self._producer = kafka.KafkaProducer( bootstrap_servers=self.brokers, max_request_size=self.max_request_size) else: self._producer = None def doShutdown(self): if self._producer: self._producer.close() def _setProducerConfig(self, **configs): self.doShutdown() self._producer = kafka.KafkaProducer(bootstrap_servers=self.brokers, **configs) def send(self, topic, message, key=None, timestamp=None, partition=None): """ Produces and flushes the provided message :param topic: Topic on which the message is to be produced :param message: Message :param key: key, for a compacted topic :param timestamp: message timestamp in milliseconds :param partition: partition on which the message is to be produced :return: """ self._producer.send(topic, message, key, partition, timestamp) self._producer.flush()
class BaseCacheClient(Device): """ An extensible read/write client for the NICOS cache. """ parameters = { 'cache': Param('"host[:port]" of the cache instance to connect to', type=host(defaultport=DEFAULT_CACHE_PORT), mandatory=True), 'prefix': Param('Cache key prefix', type=str, mandatory=True), } remote_callbacks = True _worker = None _startup_done = None def doInit(self, mode): # Should the worker connect or disconnect? self._should_connect = True # this event is set as soon as: # * the connection is established and the connect_action is done, or # * the initial connection failed # this prevents devices from polling parameter values before all values # from the cache have been received self._startup_done = threading.Event() self._connected = False self._socket = None self._secsocket = None self._sec_lock = threading.RLock() self._prefix = self.prefix.strip('/') if self._prefix: self._prefix += '/' self._selecttimeout = CYCLETIME # seconds self._do_callbacks = self.remote_callbacks self._disconnect_warnings = 0 # maps newprefix -> oldprefix without self._prefix prepended self._inv_rewrites = {} # maps oldprefix -> set of new prefixes without self._prefix prepended self._rewrites = {} self._prefixcallbacks = {} self._stoprequest = False self._queue = queue.Queue() self._synced = True # create worker thread, but do not start yet, leave that to subclasses self._worker = createThread('CacheClient worker', self._worker_thread, start=False) def _getCache(self): return None def doShutdown(self): self._stoprequest = True if self._worker and self._worker.is_alive(): self._worker.join() def _connect(self): self._do_callbacks = False self._startup_done.clear() self.log.debug('connecting to %s', self.cache) try: self._socket = tcpSocket(self.cache, DEFAULT_CACHE_PORT, timeout=5, keepalive=10) except Exception as err: self._disconnect('unable to connect to %s: %s' % (self.cache, err)) else: self.log.info('now connected to %s', self.cache) self._connected = True self._disconnect_warnings = 0 try: self._connect_action() except Exception as err: self._disconnect('unable to init connection to %s: %s' % (self.cache, err)) self._startup_done.set() self._do_callbacks = self.remote_callbacks def _disconnect(self, why=''): self._connected = False self._startup_done.clear() if why: if self._disconnect_warnings % 10 == 0: self.log.warning(why) self._disconnect_warnings += 1 if self._socket: closeSocket(self._socket) self._socket = None # close secondary socket with self._sec_lock: if self._secsocket: closeSocket(self._secsocket) self._secsocket = None self._disconnect_action() def _wait_retry(self): sleep(self._long_loop_delay) def _wait_data(self): pass def _connect_action(self): # send request for all keys and updates.... # (send a single request for a nonexisting key afterwards to # determine the end of data) msg = '@%s%s\n%s%s\n' % (self._prefix, OP_WILDCARD, END_MARKER, OP_ASK) self._socket.sendall(to_utf8(msg)) # read response data, n = b'', 0 sentinel = to_utf8(END_MARKER + OP_TELLOLD + '\n') while not data.endswith(sentinel) and n < 1000: data += self._socket.recv(BUFSIZE) n += 1 # send request for all updates msg = '@%s%s\n' % (self._prefix, OP_SUBSCRIBE) self._socket.sendall(to_utf8(msg)) for prefix in self._prefixcallbacks: msg = '@%s%s\n' % (prefix, OP_SUBSCRIBE) self._socket.sendall(to_utf8(msg)) self._process_data(data) def _disconnect_action(self): pass def _handle_msg(self, time, ttlop, ttl, tsop, key, op, value): raise NotImplementedError('implement _handle_msg in subclasses') def _process_data(self, data, sync_str=to_utf8(SYNC_MARKER + OP_TELLOLD), lmatch=line_pattern.match, mmatch=msg_pattern.match): # n = 0 i = 0 # avoid making a string copy for every line match = lmatch(data, i) while match: line = match.group(1) i = match.end() if sync_str in line: self.log.debug('process data: received sync: %r', line) self._synced = True else: msgmatch = mmatch(from_utf8(line)) # ignore invalid lines if msgmatch: # n += 1 try: self._handle_msg(**msgmatch.groupdict()) except Exception: self.log.exception('error handling message %r', msgmatch.group()) # continue loop match = lmatch(data, i) # self.log.debug('processed %d items', n) return data[i:] def _worker_thread(self): while True: try: self._worker_inner() except Exception: self.log.exception('exception in cache worker thread; ' 'restarting (please report a bug)') if self._stoprequest: break # ensure we do not restart during shutdown else: # normal termination break def _worker_inner(self): data = b'' process = self._process_data while not self._stoprequest: if self._should_connect: if not self._socket: self._connect() if not self._socket: self._wait_retry() continue else: if self._socket: self._disconnect() self._wait_retry() continue # process data so far data = process(data) # wait for a whole line of data to arrive while b'\n' not in data and self._socket and self._should_connect \ and not self._stoprequest: # optionally do some action while waiting self._wait_data() if self._queue.empty(): # NOTE: the queue.empty() check is not 100% reliable, but # that is not important here: all we care is about not # having the select always return immediately for writing writelist = [] else: writelist = [self._socket] # read or write some data while 1: try: res = select.select([self._socket], writelist, [], self._selecttimeout) except EnvironmentError as e: if e.errno == errno.EINTR: continue raise except TypeError: # socket was None, let the outer loop handle that res = ([], [], []) break if res[1]: # determine if something needs to be sent tosend = '' itemcount = 0 try: # bunch a few messages together, but not unlimited for _ in xrange(10): tosend += self._queue.get(False) itemcount += 1 except queue.Empty: pass # write data try: self._socket.sendall(to_utf8(tosend)) except Exception: self._disconnect('disconnect: send failed') # report data as processed, but then re-queue it to send # after reconnect for _ in range(itemcount): self._queue.task_done() data = b'' self._queue.put(tosend) break for _ in range(itemcount): self._queue.task_done() if res[0]: # got some data try: newdata = self._socket.recv(BUFSIZE) except Exception: newdata = b'' if not newdata: # no new data from blocking read -> abort self._disconnect('disconnect: recv failed') data = b'' break data += newdata if self._socket: # send rest of data tosend = '' itemcount = 0 try: while 1: tosend += self._queue.get(False) itemcount += 1 except queue.Empty: pass try: self._socket.sendall(to_utf8(tosend)) except Exception: self.log.debug('exception while sending last batch of updates', exc=1) # no reraise, we'll disconnect below anyways for _ in range(itemcount): self._queue.task_done() # end of while loop self._disconnect() def _single_request(self, tosend, sentinel=b'\n', retry=2, sync=False): """Communicate over the secondary socket.""" if not self._socket: self._disconnect('single request: no socket') if not self._socket: raise CacheError('cache not connected') if sync: # sync has to be false for lock requests, as these occur during startup self._queue.join() with self._sec_lock: if not self._secsocket: try: self._secsocket = tcpSocket(self.cache, DEFAULT_CACHE_PORT) except Exception as err: self.log.warning( 'unable to connect secondary socket ' 'to %s: %s', self.cache, err) self._secsocket = None self._disconnect('secondary socket: could not connect') raise CacheError('secondary socket could not be created') try: # write request # self.log.debug("get_explicit: sending %r", tosend) self._secsocket.sendall(to_utf8(tosend)) # give 10 seconds time to get the whole reply timeout = currenttime() + 10 # read response data = b'' while not data.endswith(sentinel): newdata = self._secsocket.recv(BUFSIZE) # blocking read if not newdata: raise socket.error('cache closed connection') if currenttime() > timeout: # do not just break, we need to reopen the socket raise socket.error('getting response took too long') data += newdata except socket.error: self.log.warning('error during cache query', exc=1) closeSocket(self._secsocket) self._secsocket = None if retry: for m in self._single_request(tosend, sentinel, retry - 1): yield m return raise lmatch = line_pattern.match mmatch = msg_pattern.match i = 0 # self.log.debug("get_explicit: data =%r", data) match = lmatch(data, i) while match: line = match.group(1) i = match.end() msgmatch = mmatch(from_utf8(line)) if not msgmatch: # ignore invalid lines continue # self.log.debug('line processed: %r', line) yield msgmatch match = lmatch(data, i) def waitForStartup(self, timeout): self._startup_done.wait(timeout) def flush(self): """wait for empty output queue""" self._synced = False self._queue.put('%s%s\n' % (SYNC_MARKER, OP_ASK)) self._queue.join() for _ in range(100): # self.log.debug('flush; waiting for sync...') if self._synced: break sleep(CYCLETIME) def addPrefixCallback(self, prefix, function): """Add a "prefix" callback, which is called for every key and value that does not match the prefix parameter of the client, but matches the prefix given to this function. """ if prefix not in self._prefixcallbacks: self._queue.put('@%s%s\n' % (prefix, OP_SUBSCRIBE)) self._prefixcallbacks[prefix] = function def removePrefixCallback(self, prefix): """Remove a "prefix" callback. This removes the callback previously installed by addPrefixCallback. If prefix is unknown, then do nothing. """ if prefix in self._prefixcallbacks: self._queue.put('@%s%s\n' % (prefix, OP_UNSUBSCRIBE)) del self._prefixcallbacks[prefix] # methods to make this client usable as the main device in a simple session def start(self, *args): self._connect() self._worker.start() def wait(self): while not self._stoprequest: sleep(self._long_loop_delay) if self._worker and self._worker.is_alive(): self._worker.join() def quit(self, signum=None): self.log.info('quitting on signal %s...', signum) self._stoprequest = True def lock(self, key, ttl=None, unlock=False, sessionid=None): """Locking/unlocking: opens a separate connection.""" tosend = '%s%s%s%s%s\n' % (self._prefix, key.lower(), OP_LOCK, unlock and OP_LOCK_UNLOCK or OP_LOCK_LOCK, sessionid or session.sessionid) if ttl is not None: tosend = ('+%s@' % ttl) + tosend for msgmatch in self._single_request(tosend, sync=False): if msgmatch.group('value'): raise CacheLockError(msgmatch.group('value')) return # no response received; let's assume standalone mode self.log.warning('allowing lock/unlock operation without cache ' 'connection') def unlock(self, key, sessionid=None): return self.lock(key, ttl=None, unlock=True, sessionid=sessionid) def storeSysInfo(self, service): """Store info about the service in the cache.""" if not self._socket: return try: key, res = getSysInfo(service) msg = '%s@%s%s%s\n' % (currenttime(), key, OP_TELL, cache_dump(res)) self._socket.sendall(to_utf8(msg)) except Exception: self.log.exception('storing sysinfo failed')
class PixelmanUDPChannel(ActiveChannel): """ Trigger detector controlled by Pixelman software via UDP service One of the detectors at V20 can be triggered through a simple UDP service which listens on the computer that controls the detector. It expects a keyword to start acquisition and then sends a keyword back once it's done. The service expects a new connection for each data acquisition, so the connection is established in the doStart method and removed in doFinish. In order to recover from inconsistent states, the socket is also torn down in doStop, although that won't stop the detector, just reset the connection. At the moment it's not possible to obtain count information from the service, but that may change in the future. """ parameters = { 'host': Param('IP and port for Pixelman Detector UDP interface.', type=host()), 'acquire': Param('Keyword to send for starting the acquisition', type=str), 'finished': Param( 'Keyword to wait for to determine ' 'whether the acquisition is done', type=str), 'acquiring': Param('Internal parameter to synchronise between processes.', type=bool, internal=True, default=False, mandatory=False, settable=False) } parameter_overrides = {'ismaster': Override(default=True, settable=True)} def valueInfo(self): return Value(self.name, unit=self.unit, type='other', fmtstr=self.fmtstr), def doInit(self, mode): self._socket = None def doStart(self): if self._socket is None: self.log.debug('Socket is None, creating socket.') pm_host, pm_port = self.host.split(':') self.log.debug('Connection: ' + self.host) self._socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) self._socket.connect((pm_host, int(pm_port))) self.log.debug('Sending Keyword: ' + self.acquire) self._socket.sendall(self.acquire) self._socket.setblocking(0) self._setROParam('acquiring', True) self.log.debug('Acquisition started') else: self.log.info( 'Socket already exists, starting again has no effect.') def doFinish(self): self.log.debug('Finishing...') if self._socket is not None: self.log.debug('Actually shutting down...') self._socket.close() self._socket = None self._setROParam('acquiring', False) def doRead(self, maxage=0): return 0 def doStop(self): self.doFinish() def doStatus(self, maxage=0): if not self._check_complete(): return status.BUSY, 'Acquiring...' return status.OK, 'Idle' def duringMeasureHook(self, elapsed): return None def _check_complete(self): self.log.debug('Checking completion...') if session.sessiontype != POLLER: if self._socket is not None: self.log.debug('Actually performing check...') try: data = self._socket.recv(1024) self.log.debug('Got data: ' + data) return data == self.finished except socket.error: return False self.log.debug('Falling back to Cache...') return not self.acquiring
class KafkaSubscriber(DeviceMixinBase): """ Receives messages from Kafka, can subscribe to a topic and get all new messages from the topic if required via a callback method *new_message_callback*. """ parameters = { 'brokers': Param('List of kafka hosts to be connected', type=listof(host(defaultport=9092)), mandatory=True, preinit=True, userparam=False) } def doPreinit(self, mode): if mode != SIMULATION: self._consumer = kafka.KafkaConsumer( bootstrap_servers=self.brokers, auto_offset_reset='latest' # start at latest offset ) else: self._consumer = None # Settings for thread to fetch new message self._stoprequest = True self._updater_thread = None def doShutdown(self): if self._updater_thread is not None: self._stoprequest = True if self._updater_thread.is_alive(): self._updater_thread.join() if self._consumer: self._consumer.close() @property def consumer(self): return self._consumer def subscribe(self, topic): """ Create the thread that provides call backs on new messages """ # Remove all the assigned topics self._consumer.unsubscribe() topics = self._consumer.topics() if topic not in topics: raise ConfigurationError('Provided topic %s does not exist' % topic) # Assign the partitions partitions = self._consumer.partitions_for_topic(topic) if not partitions: raise ConfigurationError('Cannot query partitions for %s' % topic) self._consumer.assign([kafka.TopicPartition(topic, p) for p in partitions]) self._stoprequest = False self._updater_thread = createThread('updater_' + topic, self._get_new_messages) self.log.debug('subscribed to updates from topic: %s' % topic) def _get_new_messages(self): while not self._stoprequest: sleep(self._long_loop_delay) messages = {} data = self._consumer.poll(5) for records in data.values(): for record in records: messages[record.timestamp] = record.value if messages: self.new_messages_callback(messages) else: self.no_messages_callback() self.log.debug("KafkaSubscriber thread finished") def new_messages_callback(self, messages): """This method is called whenever a new messages appear on the topic. The subclasses should define this method if a callback is required when new messages appear. :param messages: dict of timestamp and raw message """ def no_messages_callback(self): """This method is called if no messages are on the topic.
class KafkaSubscriber(DeviceMixinBase): """ Receives messages from Kafka, can subscribe to a topic and get all new messages from the topic if required via a callback method *new_message_callback*. """ parameters = { 'brokers': Param('List of kafka hosts to be connected', type=listof(host(defaultport=9092)), default=['localhost'], preinit=True, userparam=False) } def doPreinit(self, mode): self._consumer = kafka.KafkaConsumer( bootstrap_servers=self.brokers, auto_offset_reset='latest' # start at latest offset ) # Settings for thread to fetch new message self._stoprequest = True self._updater_thread = None def doShutdown(self): if self._updater_thread is not None: self._stoprequest = True if self._updater_thread.is_alive(): self._updater_thread.join() self._consumer.close() @property def consumer(self): return self._consumer def subscribe(self, topic): """ Create the thread that provides call backs on new messages """ # Remove all the assigned topics self._consumer.unsubscribe() self._consumer.subscribe(topic) # # Assign the partitions # partitions = self._consumer.partitions_for_topic(topic) # if not partitions: # raise NicosError('Provided topic %s does not exist' % topic) # # self._consumer.assign([kafka.TopicPartition(topic, p) # for p in partitions]) # self._stoprequest = False # self._updater_thread = createThread('updater_' + topic, # self._get_new_messages) self.log.debug('subscribed to updates from topic: %s' % topic) def _get_new_messages(self): while not self._stoprequest: sleep(self._long_loop_delay) assignment = self._consumer.assignment() messages = {} end = self._consumer.end_offsets(list(assignment)) for p in assignment: while self._consumer.position(p) < end[p]: msg = next(self._consumer) messages[msg.timestamp] = msg.value self.new_messages_callback(messages) def new_messages_callback(self, messages): """This method is called whenever a new messages appear on the topic. The subclasses should define this method if a callback is required when new messages appear. :param messages: dict of timestamp and raw message """ pass
class NicosDaemon(Device): """ This class abstracts the main daemon process. """ attached_devices = { 'authenticators': Attach( 'The authenticator devices to use for ' 'validating users and passwords', Authenticator, multiple=True), } parameters = { 'server': Param('Address to bind to (host or host[:port])', type=host(defaultport=DEFAULT_PORT), mandatory=True, ext_desc='The default port is ``1301``.'), 'servercls': Param('Server class used for creating transports ' 'to each client', type=str, mandatory=False, default='nicos.services.daemon.proto.classic.' 'Server'), 'serializercls': Param( 'Serializer class used for serializing ' 'messages transported from/to the server', type=str, mandatory=False, default='nicos.protocols.daemon.classic.' 'ClassicSerializer'), 'maxlogins': Param('Maximum number of simultaneous clients ' 'served', type=int, default=10), 'updateinterval': Param( 'Interval for watch expressions checking and' ' sending updates to the clients', type=float, unit='s', default=0.2), 'trustedhosts': Param('A list of trusted hosts allowed to log in', type=listof(str), ext_desc='An empty list means all hosts are ' 'allowed.'), 'simmode': Param('Whether to always start in dry run mode', type=bool), 'autosimulate': Param('Whether to simulate scripts when running them', type=bool, default=False) } def doInit(self, mode): # import server and serializer class servercls = importString(self.servercls) serialcls = importString(self.serializercls) self._stoprequest = False # the controller represents the internal script execution machinery if self.autosimulate and not config.sandbox_simulation: raise ConfigurationError('autosimulation configured but sandbox' ' deactivated') self._controller = ExecutionController(self.log, self.emit_event, 'startup', self.simmode, self.autosimulate) # cache log messages emitted so far self._messages = [] host, port = parseHostPort(self.server, DEFAULT_PORT) # create server (transport + serializer) self._server = servercls(self, (host, port), serialcls()) self._watch_worker = createThread('daemon watch monitor', self._watch_entry) def _watch_entry(self): """ This thread checks for watch value changes periodically and sends out events on changes. """ # pre-fetch attributes for speed ctlr, intv, emit, sleep = self._controller, self.updateinterval, \ self.emit_event, time.sleep lastwatch = {} while not self._stoprequest: sleep(intv) # new watch values? watch = ctlr.eval_watch_expressions() if watch != lastwatch: emit('watch', watch) lastwatch = watch def emit_event(self, event, data, blobs=None): """Emit an event to all handlers.""" self._server.emit(event, data, blobs or []) def emit_event_private(self, event, data, blobs=None): """Emit an event to only the calling handler.""" handler = self._controller.get_current_handler() if handler: self._server.emit(event, data, blobs or [], handler=handler) def statusinfo(self): self.log.info('got SIGUSR2 - current stacktraces for each thread:') active = threading._active for tid, frame in list(sys._current_frames().items()): if tid in active: name = active[tid].getName() else: name = str(tid) self.log.info('%s: %s', name, formatExtendedStack(frame)) def start(self): """Start the daemon's server.""" self.log.info('NICOS daemon v%s started, starting server on %s', nicos_version, self.server) # startup the script thread self._controller.start_script_thread() self._worker = createThread('daemon server', self._server.start, args=(self._long_loop_delay, )) def wait(self): while not self._stoprequest: time.sleep(self._long_loop_delay) self._worker.join() def quit(self, signum=None): self.log.info('quitting on signal %s...', signum) self._stoprequest = True self._server.stop() self._worker.join() self._server.close() def current_script(self): return self._controller.current_script def current_user(self): return getattr(self._controller.thread_data, 'user', system_user) def get_authenticators(self): return self._attached_authenticators
class KafkaCacheDatabase(MemoryCacheDatabase): """ Cache database that stores cache in Kafka topics without History. Current key and value pairs are stored in Kafka using log compaction. The `CacheEntry` values can be serialized using the attached device. This is then used to encode and decode entry instances while producing to the Kafka topic and while consuming from it. The data is stored in the Kafka topics using log compaction. Log compaction ensures that Kafka will always retain at least the last known value for each message key within the log of data for a single topic partition. The keys for partitioning are the keys received from cache server and the timestamp of the message is same as the time of the cache entry. If the provided `topic` does not exist, the cache database will not be able to proceed. Use the command line kafka tool to create the topic. This database will connect to the kafka and zookeeper services so they should be running in background on the provided `hosts`. For a basic getting started with kafka one can use the following guide: https://kafka.apache.org/quickstart Note that the minimum kafka broker version required for this database to run properly is 0.11.0. Before this version timestamps for messages could not be set. The default behavior of Kafka does not ensure that the topic will be cleaned up using the log compaction policy. For this to happen one of the following two is to be done: * Create topic by CLI Kafka tools using --config cleanup.policy=compact * Set in server.properties log.cleanup.policy=compact History is NOT supported with this database. """ parameters = { 'currenttopic': Param('Kafka topic where the current values of cache are streamed', type=str, mandatory=True), 'brokers': Param('List of Kafka bootstrap servers.', type=listof(host(defaultport=9092)), default=['localhost']), } attached_devices = { 'serializer': Attach('Device to serialize the cache entry values', CacheEntrySerializer, optional=False) } def doInit(self, mode): MemoryCacheDatabase.doInit(self, mode) # Create the producer self._producer = KafkaProducer(bootstrap_servers=self.brokers) # Create the consumer self._consumer = KafkaConsumer( bootstrap_servers=self.brokers, auto_offset_reset='earliest' # start at earliest topic ) # Give up if the topic does not exist if self.currenttopic not in self._consumer.topics(): raise ConfigurationError( 'Topic "%s" does not exit. Create this topic and restart.' % self.currenttopic) # Assign the partitions partitions = self._consumer.partitions_for_topic(self.currenttopic) self._consumer.assign( [TopicPartition(self.currenttopic, p) for p in partitions]) # Cleanup thread configuration self._stoprequest = False self._cleaner = createThread('cleaner', self._clean, start=False) def doShutdown(self): self._consumer.close() self._producer.close() # Stop the cleaner thread self._stoprequest = True self._cleaner.join() def initDatabase(self): self.log.info('Reading messages from kafka topic - %s', self.currenttopic) now = currenttime() message_count = 0 end = self._consumer.end_offsets(list(self._consumer.assignment())) for partition in self._consumer.assignment(): while self._consumer.position(partition) < end[partition]: msg = next(self._consumer) message_count += 1 if msg.value is not None: _, entry = self._attached_serializer.decode(msg.value) if entry is not None and entry.value is not None: # self.log.debug('%s (%s): %s -> %s', msg.offset, # msg.timestamp, msg.key, entry) if entry.ttl and entry.time + entry.ttl < now: entry.expired = True self._db[msg.key] = [entry] self._cleaner.start() self.log.info('Processed %i messages.', message_count) def _clean(self): def cleanonce(): with self._db_lock: for key, entries in iteritems(self._db): entry = entries[-1] if not entry.value or entry.expired: continue time = currenttime() if entry.ttl and (entry.time + entry.ttl < time): entry.expired = True for client in self._server._connected.values(): client.update(key, OP_TELLOLD, entry.value, time, None) while not self._stoprequest: sleep(self._long_loop_delay) cleanonce() def _update_topic(self, key, entry): # This method is responsible to communicate and update all the # topics that should be updated. Subclasses can (re)implement it # if there are messages to be produced to other topics self.log.debug('Writing: %s -> %s', key, entry.value) # For the log-compacted topic key deletion happens when None is # passed as the value for the key value = None if entry.value is not None: # Only when the key deletion is not required value = self._attached_serializer.encode(key, entry) self._producer.send(topic=self.currenttopic, value=value, key=bytes(key), timestamp_ms=int(entry.time * 1000)) # clear all local buffers and produce pending messages self._producer.flush() def tell(self, key, value, time, ttl, from_client): if value is None: # deletes cannot have a TTL ttl = None send_update = True always_send_update = False # remove no-store flag if key.endswith(FLAG_NO_STORE): key = key[:-len(FLAG_NO_STORE)] always_send_update = True try: category, subkey = key.rsplit('/', 1) except ValueError: category = 'nocat' subkey = key newcats = [category] if category in self._rewrites: newcats.extend(self._rewrites[category]) for newcat in newcats: key = newcat + '/' + subkey with self._db_lock: entries = self._db.setdefault(key, []) if entries: lastent = entries[-1] if lastent.value == value and not lastent.expired: # not a real update send_update = False thisent = CacheEntry(time, ttl, value) entries[:] = [thisent] if send_update: self._update_topic(key, thisent) if send_update or always_send_update: for client in self._server._connected.values(): if client is not from_client and client.is_active(): client.update(key, OP_TELL, value or '', time, ttl)
class CacheServer(Device): """ The server class. """ parameters = { 'server': Param('Address to bind to (host or host:port)', type=host(defaultport=DEFAULT_CACHE_PORT), mandatory=True, ext_desc="The default port is ``14869``."), } attached_devices = { 'db': Attach('The cache database instance', CacheDatabase), } def doInit(self, mode): self._stoprequest = False # TCP server address if bound to TCP self._boundto = None # server sockets for TCP and UDP self._serversocket = None self._serversocket_udp = None # worker connections self._connected = {} self._attached_db._server = self self._connectionLock = threading.Lock() def start(self, *startargs): if config.instrument == 'demo' and 'clear' in startargs: self._attached_db.clearDatabase() self._attached_db.initDatabase() self.storeSysInfo() self._worker = createThread('server', self._server_thread) def storeSysInfo(self): key, res = getSysInfo('cache') self._attached_db.tell(key, str(res), currenttime(), None, None) def _bind_to(self, address, proto='tcp'): # bind to the address with the given protocol; return socket and address host, port = parseHostPort(address, DEFAULT_CACHE_PORT) serversocket = socket.socket( socket.AF_INET, proto == 'tcp' and socket.SOCK_STREAM or socket.SOCK_DGRAM) serversocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) if proto == 'udp': # we want to be able to receive UDP broadcasts serversocket.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) try: serversocket.bind((socket.gethostbyname(host), port)) if proto == 'tcp': serversocket.listen(50) # max waiting connections.... return serversocket, (host, port) except Exception: serversocket.close() return None, None # failed, return None as indicator def _server_thread(self): self.log.info('server starting') # bind UDP broadcast socket self.log.debug('trying to bind to UDP broadcast') self._serversocket_udp = self._bind_to('', 'udp')[0] if self._serversocket_udp: self.log.info('UDP bound to broadcast') # now try to bind TCP socket, include 'MUST WORK' standalone names self.log.debug('trying to bind to %s', self.server) self._serversocket, self._boundto = self._bind_to(self.server) # one of the must have worked, otherwise continuing makes no sense if not self._serversocket and not self._serversocket_udp: self._stoprequest = True self.log.error("couldn't bind any sockets, giving up!") return if not self._boundto: self.log.warning('starting main loop only bound to UDP broadcast') else: self.log.info('TCP bound to %s:%s', self._boundto[0], self._boundto[1]) # now enter main serving loop while not self._stoprequest: # loop through connections, first to remove dead ones, # secondly to try to reconnect for addr, client in list(self._connected.items()): if not client.is_active(): # dead or stopped self.log.info('client connection %s closed', addr) client.closedown() client.join() # wait for threads to end del self._connected[addr] # now check for additional incoming connections # build list of things to check selectlist = [] if self._serversocket: selectlist.append(self._serversocket) if self._serversocket_udp: selectlist.append(self._serversocket_udp) # 3 times client-side timeout res = select.select(selectlist, [], [], CYCLETIME * 3) if not res[0]: continue # nothing to read -> continue loop # lock aginst code in self.quit with self._connectionLock: if self._stoprequest: break if self._serversocket in res[0]: # TCP connection came in conn, addr = self._serversocket.accept() addr = 'tcp://%s:%d' % addr self.log.info('new connection from %s', addr) self._connected[addr] = CacheWorker(self._attached_db, conn, name=addr, loglevel=self.loglevel) elif self._serversocket_udp in res[0]: # UDP data came in data, addr = self._serversocket_udp.recvfrom(3072) nice_addr = 'udp://%s:%d' % addr self.log.info('new connection from %s', nice_addr) self._connected[nice_addr] = CacheUDPWorker( self._attached_db, self._serversocket_udp, name=nice_addr, data=data, remoteaddr=addr, loglevel=self.loglevel) if self._serversocket: closeSocket(self._serversocket) self._serversocket = None def wait(self): while not self._stoprequest: sleep(self._long_loop_delay) self._worker.join() def quit(self, signum=None): self.log.info('quitting on signal %s...', signum) self._stoprequest = True # without locking, the _connected list may not have all clients yet.... with self._connectionLock: for client in list(self._connected.values()): self.log.info('closing client %s', client) if client.is_active(): client.closedown() with self._connectionLock: for client in list(self._connected.values()): self.log.info('waiting for %s', client) client.closedown() # make sure, the connection closes down client.join() self.log.info('waiting for server') self._worker.join() self.log.info('server finished')
class JustBinItDetector(Detector): """ A "detector" that reads image data from just-bin-it. Note: it only uses image channels. """ parameters = { 'brokers': Param('List of kafka hosts to be connected', type=listof(host(defaultport=9092)), mandatory=True, preinit=True, userparam=False), 'command_topic': Param( 'The topic to send just-bin-it commands to', type=str, userparam=False, settable=False, mandatory=True, ), 'response_topic': Param( 'The topic where just-bin-it responses appear', type=str, userparam=False, settable=False, mandatory=True, ), 'ack_timeout': Param( 'How long to wait for timeout on acknowledgement', type=int, default=5, unit='s', userparam=False, settable=False, ), } parameter_overrides = { 'unit': Override(default='events', settable=False, mandatory=False), 'fmtstr': Override(default='%d'), 'liveinterval': Override(type=floatrange(0.5), default=1), } _last_live = 0 _presets = {} _presetkeys = {'t'} _ack_thread = None _exit_thread = False def doPreinit(self, mode): self._command_sender = kafka.KafkaProducer( bootstrap_servers=self.brokers) # Set up the response message consumer self._response_consumer = kafka.KafkaConsumer( bootstrap_servers=self.brokers) self._response_topic = kafka.TopicPartition(self.response_topic, 0) self._response_consumer.assign([self._response_topic]) self._response_consumer.seek_to_end() self.log.debug('Response topic consumer initial position = %s', self._response_consumer.position(self._response_topic)) def doInit(self, _mode): pass def doPrepare(self): for image_channel in self._attached_images: image_channel.doPrepare() def doStart(self): self._last_live = -(self.liveinterval or 0) # Generate a unique-ish id unique_id = 'nicos-{}-{}'.format(self.name, int(time.time())) self.log.debug('set unique id = %s', unique_id) count_interval = self._presets.get('t', None) config = self._create_config(count_interval, unique_id) if count_interval: self.log.info( 'Requesting just-bin-it to start counting for %s seconds', count_interval) else: self.log.info('Requesting just-bin-it to start counting') self._send_command(self.command_topic, json.dumps(config).encode()) # Tell the channels to start for image_channel in self._attached_images: image_channel.doStart() # Check for acknowledgement of the command being received self._exit_thread = False self._ack_thread = createThread("jbi-ack", self._check_for_ack, (unique_id, self.ack_timeout)) def _check_for_ack(self, identifier, timeout_duration): timeout = int(time.time()) + timeout_duration acknowledged = False while not (acknowledged or self._exit_thread): messages = self._response_consumer.poll(timeout_ms=50) responses = messages.get(self._response_topic, []) for records in responses: msg = json.loads(records.value) if 'msg_id' in msg and msg['msg_id'] == identifier: acknowledged = self._handle_message(msg) break # Check for timeout if not acknowledged and int(time.time()) > timeout: err_msg = 'Count aborted as no acknowledgement received from ' \ 'just-bin-it within timeout duration '\ f'({timeout_duration} seconds)' self.log.error(err_msg) break if not acknowledged: # Couldn't start histogramming, so stop the channels etc. self._stop_histogramming() for image_channel in self._attached_images: image_channel.doStop() def _handle_message(self, msg): if 'response' in msg and msg['response'] == 'ACK': self.log.info('Counting request acknowledged by just-bin-it') return True elif 'response' in msg and msg['response'] == 'ERR': self.log.error('just-bin-it could not start counting: %s', msg['message']) else: self.log.error( 'Unknown response message received from just-bin-it') return False def _send_command(self, topic, message): self._command_sender.send(topic, message) self._command_sender.flush() def _create_config(self, interval, identifier): histograms = [] for image_channel in self._attached_images: histograms.append(image_channel.get_configuration()) config_base = { 'cmd': 'config', 'msg_id': identifier, 'histograms': histograms } if interval: config_base['interval'] = interval else: # If no interval then start open-ended count config_base['start'] = int(time.time()) * 1000 return config_base def valueInfo(self): return tuple(info for channel in self._attached_images for info in channel.valueInfo()) def doRead(self, maxage=0): return [ data for channel in self._attached_images for data in channel.read(maxage) ] def doReadArrays(self, quality): return [image.doReadArray(quality) for image in self._attached_images] def doFinish(self): self._stop_ack_thread() self._stop_histogramming() def _stop_histogramming(self): self._send_command(self.command_topic, b'{"cmd": "stop"}') def doShutdown(self): self._response_consumer.close() def doSetPreset(self, **presets): self._presets = presets def doStop(self): self._stop_ack_thread() self._stop_histogramming() def _stop_ack_thread(self): if self._ack_thread and self._ack_thread.is_alive(): self._exit_thread = True self._ack_thread.join() self._ack_thread = None def doStatus(self, maxage=0): return multiStatus(self._attached_images, maxage) def doReset(self): pass def doInfo(self): return [ data for channel in self._attached_images for data in channel.doInfo() ] def duringMeasureHook(self, elapsed): if elapsed > self._last_live + self.liveinterval: self._last_live = elapsed return LIVE return None def arrayInfo(self): return tuple(image.arrayInfo() for image in self._attached_images)
class CacheKafkaForwarder(ForwarderBase, Device): parameters = { 'brokers': Param('List of kafka hosts to be connected', type=listof(host(defaultport=9092)), mandatory=True, preinit=True, userparam=False ), 'output_topic': Param('The topic to send data to', type=str, userparam=False, settable=False, mandatory=True, ), 'dev_ignore': Param('Devices to ignore; if empty, all devices are ' 'accepted', default=[], type=listof(str), ), 'update_interval': Param('Time interval (in secs.) to send regular updates', default=10.0, type=float, settable=False) } parameter_overrides = { # Key filters are irrelevant for this collector 'keyfilters': Override(default=[], settable=False), } def doInit(self, mode): self._dev_to_value_cache = {} self._dev_to_status_cache = {} self._dev_to_timestamp_cache = {} self._producer = None self._lock = Lock() self._initFilters() self._queue = queue.Queue(1000) self._worker = createThread('cache_to_kafka', self._processQueue, start=False) self._regular_update_worker = createThread( 'send_regular_updates', self._poll_updates, start=False) while not self._producer: try: self._producer = \ KafkaProducer(bootstrap_servers=self._config['brokers']) except Exception as error: self.log.error( 'Could not connect to Kafka - will try again soon: %s', error) time.sleep(5) self.log.info('Connected to Kafka brokers %s', self._config['brokers']) def _startWorker(self): self._worker.start() self._regular_update_worker.start() def _poll_updates(self): while True: with self._lock: for dev_name in set(self._dev_to_value_cache.keys()).union( self._dev_to_status_cache.keys()): if self._relevant_properties_available(dev_name): self._push_to_queue( dev_name, self._dev_to_value_cache[dev_name], self._dev_to_status_cache[dev_name], self._dev_to_timestamp_cache[dev_name]) time.sleep(self.update_interval) def _checkKey(self, key): if key.endswith('/value') or key.endswith('/status'): return True return False def _checkDevice(self, name): if name not in self.dev_ignore: return True return False def _putChange(self, timestamp, ttl, key, value): if value is None: return dev_name = key[0:key.index('/')] if not self._checkKey(key) or not self._checkDevice(dev_name): return self.log.debug('_putChange %s %s %s', key, value, time) with self._lock: if key.endswith('value'): self._dev_to_value_cache[dev_name] = value else: self._dev_to_status_cache[dev_name] = convert_status(value) timestamp_ns = int(float(timestamp) * 10 ** 9) self._dev_to_timestamp_cache[dev_name] = timestamp_ns # Don't send until have at least one reading for both value and status if self._relevant_properties_available(dev_name): self._push_to_queue( dev_name, self._dev_to_value_cache[dev_name], self._dev_to_status_cache[dev_name], timestamp_ns,) def _relevant_properties_available(self, dev_name): return dev_name in self._dev_to_value_cache \ and dev_name in self._dev_to_status_cache \ and dev_name in self._dev_to_timestamp_cache def _push_to_queue(self, dev_name, value, status, timestamp): try: self._queue.put_nowait((dev_name, value, status, timestamp)) except queue.Full: self.log.error('Queue full, so discarding older value(s)') self._queue.get() self._queue.put((dev_name, value, status, timestamp)) self._queue.task_done() def _processQueue(self): while True: name, value, status, timestamp = self._queue.get() try: # Convert value from string to correct type value = cache_load(value) if not isinstance(value, str): # Policy decision: don't send strings via f142 buffer = to_f142(name, value, status, timestamp) self._send_to_kafka(buffer) except Exception as error: self.log.error('Could not forward data: %s', error) self._queue.task_done() def doShutdown(self): self._producer.close() def _send_to_kafka(self, buffer): self._producer.send(self.output_topic, buffer) self._producer.flush(timeout=3)