def process_request(self, request, client_address, client_id):
"""Process a "request", that is, a client connection."""
# mostly copied from ThreadingMixIn but without the import,
# which causes threading issues because of the import lock
createThread('request handler',
self.process_request_thread,
args=(request, client_address, client_id))
def start(self, options): # pylint: disable=W0221
self.log.info('monitor starting up, creating main window')
self._fontsize = options.fontsize or self.fontsize
self._fontsizebig = int(self._fontsize * 1.2)
self._timefontsize = (options.timefontsize or self.timefontsize
or (self._fontsizebig + self._fontsize))
self._padding = options.padding or self.padding
self._geometry = options.geometry or self.geometry
if self._geometry and self._geometry != 'fullscreen':
try:
m = re.match(r'(?:(\d+)x(\d+))?\+(\d+)\+(\d+)', self._geometry)
w, h, x, y = m.groups()
if w is None:
w = h = 0
else:
w, h = int(w), int(h)
x, y = int(x), int(y)
self._geometry = (w, h, x, y)
except Exception:
self.log.warning('invalid geometry %s', self._geometry)
self._geometry = None
# timeout for select() call
self._selecttimeout = 0.2
# maps keys to field-dicts defined in self.layout (see above)
self._keymap = {}
# blocks to hide conditionally
self._onlyblocks = []
# fields to hide conditionally (only if not in a block)
self._onlyfields = []
# remembers loaded setups
self._setups = set()
# master active?
self._masteractive = False
# currently shown warnings
self._currwarnings = ''
# start a thread checking for modification of the setup file
createThread('refresh checker', self._checker)
self.initGui()
# now start the worker thread
self._worker.start()
self.log.info('starting main loop')
try:
self.mainLoop()
except KeyboardInterrupt:
pass
self._stoprequest = True
def ftpserver():
"""Provide a ftp server with virtual files"""
handler = FTPTestHandler
handler.abstracted_fs = MyTestFS
authorizer = MyDummyAuthorizer()
home = os.curdir
authorizer.add_user('user', '12345', home, perm='elrmwM')
handler.authorizer = authorizer
server = ThreadedFTPServer(('localhost', 12345), handler)
createThread('FTP', server.serve_forever)
yield handler
server.close_all()
def start(self, setup=None):
self._setup = setup
if setup is None:
return self._start_master()
self.log.info('%s poller starting', setup)
if setup == '[dummy]':
return
try:
session.loadSetup(setup, allow_startupcode=False)
for devname in session.getSetupInfo()[setup]['devices']:
if devname in self.blacklist:
self.log.debug('not polling %s, it is blacklisted',
devname)
continue
# import the device class in the main thread; this is necessary
# for some external modules like Epics
self.log.debug('importing device class for %s', devname)
try:
session.importDevice(devname)
except Exception:
self.log.warning(
'%-10s: error importing device class, '
'not retrying this device',
devname,
exc=True)
continue
self.log.debug('starting thread for %s', devname)
queue = Queue.Queue()
worker = createThread('%s poller' % devname,
self._worker_thread,
args=(devname, queue))
worker.queue = queue
self._workers[devname.lower()] = worker
# start staggered to not poll all devs at once....
# use just a small delay, exact value does not matter
sleep(0.0719)
session.cache.addPrefixCallback('poller', self.enqueue_params_poll)
except ConfigurationError as err:
self.log.warning('Setup %r has failed to load!', setup)
self.log.error(err, exc=True)
self.log.warning('Not polling any devices!')
# start a thread checking for modification of the setup file
createThread('refresh checker', self._checker, args=(setup, ))
self.log.info('%s poller startup complete', setup)
def doInit(self, mode):
# import server and serializer class
servercls = importString(self.servercls)
serialcls = importString(self.serializercls)
self._stoprequest = False
# the controller represents the internal script execution machinery
if self.autosimulate and not config.sandbox_simulation:
raise ConfigurationError('autosimulation configured but sandbox'
' deactivated')
self._controller = ExecutionController(self.log, self.emit_event,
'startup', self.simmode,
self.autosimulate)
# cache log messages emitted so far
self._messages = []
host, port = parseHostPort(self.server, DEFAULT_PORT)
# create server (transport + serializer)
self._server = servercls(self, (host, port), serialcls())
self._watch_worker = createThread('daemon watch monitor',
self._watch_entry)
def doStart(self):
self._last_live = -(self.liveinterval or 0)
# Generate a unique-ish id
unique_id = 'nicos-{}-{}'.format(self.name, int(time.time()))
self.log.debug('set unique id = %s', unique_id)
count_interval = self._presets.get('t', None)
config = self._create_config(count_interval, unique_id)
if count_interval:
self.log.info(
'Requesting just-bin-it to start counting for %s seconds',
count_interval)
else:
self.log.info('Requesting just-bin-it to start counting')
self._send_command(self.command_topic, json.dumps(config).encode())
# Tell the channels to start
for image_channel in self._attached_images:
image_channel.doStart()
# Check for acknowledgement of the command being received
self._exit_thread = False
self._ack_thread = createThread("jbi-ack", self._check_for_ack,
(unique_id, self.ack_timeout))
def send(self, subject, body, what=None, short=None, important=True):
def send():
receivers = self._getAllRecipients(important)
if not receivers:
return
ret = sendMail(self.mailserver, receivers, self.sender,
self.subject + ' -- ' + subject, body)
if not ret: # on error, ret is a list of errors
self.log.info('%smail sent to %s', what and what + ' ' or '',
', '.join(receivers))
else:
self.log.warning('sending mail failed: %s', ', '.join(ret))
if not self._checkRateLimit():
return
createThread('mail sender', send)
def doReference(self, *args):
refswitch = args[0] if args and isinstance(args[0], string_types) \
else None
if self.doStatus()[0] == status.BUSY:
self.stop()
self.wait()
# The sixfold and triple cards will be reset as whole card, so values
# of all channels will reset !!!
# self.reset()
# self.wait()
if self.doStatus()[0] == status.OK:
if self._refcontrol and self._refcontrol.is_alive():
self._refcontrol.join()
self._refcontrol = None
if self._refcontrol is None:
threadname = 'referencing %s' % self
self._refcontrol = createThread(threadname,
self._reference,
args=(refswitch, ))
session.delay(0.2)
else:
raise NicosError(self, 'in error or busy state')
def doStart(self, target):
"""Start movement of the axis to target."""
if self._checkTargetPosition(self.read(0), target, error=False):
self.log.debug('not moving, already at %.4f within precision',
target)
return
if self._mode == SIMULATION:
self._attached_motor.start(target + self.offset)
if self._hascoder:
self._attached_coder._sim_setValue(target + self.offset)
return
if self.status(0)[0] == status.BUSY:
self.log.debug('need to stop axis first')
self.stop()
waitForCompletion(self, ignore_errors=True)
if self._posthread:
if self._posthread.is_alive():
self._posthread.join()
self._posthread = None
self._target = target
self._stoprequest = 0
self._errorstate = None
if not self._posthread:
self._posthread = createThread('positioning thread %s' % self,
self._positioningThread)
def doInit(self, mode):
self._cat = {}
self._cat_lock = threading.Lock()
CacheDatabase.doInit(self, mode)
if self.makelinks == 'auto':
# Windows compatibility: it does not provide os.link
if not hasattr(os, 'link'):
self._make_link = lambda a, b: None
else:
self._make_link = os.link
elif self.makelinks == 'hard':
self._make_link = os.link
elif self.makelinks == 'soft':
self._make_link = os.symlink
else:
self._make_link = lambda a, b: None
self._basepath = path.join(config.nicos_root, self.storepath)
ltime = localtime()
self._year = str(ltime[0])
self._currday = '%02d-%02d' % ltime[1:3]
self._midnight = mktime(ltime[:3] + (0, ) * (8 - 3) + (ltime[8], ))
self._nextmidnight = self._midnight + 86400
self._stoprequest = False
self._cleaner = createThread('cleaner', self._clean)
def doInit(self, mode):
MemoryCacheDatabase.doInit(self, mode)
# Create the producer
self._producer = KafkaProducer(bootstrap_servers=self.brokers)
# Create the consumer
self._consumer = KafkaConsumer(
bootstrap_servers=self.brokers,
auto_offset_reset='earliest' # start at earliest topic
)
# Give up if the topic does not exist
if self.currenttopic not in self._consumer.topics():
raise ConfigurationError(
'Topic "%s" does not exit. Create this topic and restart.' %
self.currenttopic)
# Assign the partitions
partitions = self._consumer.partitions_for_topic(self.currenttopic)
self._consumer.assign(
[TopicPartition(self.currenttopic, p) for p in partitions])
# Cleanup thread configuration
self._stoprequest = False
self._cleaner = createThread('cleaner', self._clean, start=False)
def doInit(self, mode):
# Should the worker connect or disconnect?
self._should_connect = True
# this event is set as soon as:
# * the connection is established and the connect_action is done, or
# * the initial connection failed
# this prevents devices from polling parameter values before all values
# from the cache have been received
self._startup_done = threading.Event()
self._connected = False
self._socket = None
self._secsocket = None
self._sec_lock = threading.RLock()
self._prefix = self.prefix.strip('/')
if self._prefix:
self._prefix += '/'
self._selecttimeout = CYCLETIME # seconds
self._do_callbacks = self.remote_callbacks
self._disconnect_warnings = 0
# maps newprefix -> oldprefix without self._prefix prepended
self._inv_rewrites = {}
# maps oldprefix -> set of new prefixes without self._prefix prepended
self._rewrites = {}
self._prefixcallbacks = {}
self._stoprequest = False
self._queue = queue.Queue()
self._synced = True
# create worker thread, but do not start yet, leave that to subclasses
self._worker = createThread('CacheClient worker',
self._worker_thread,
start=False)
def start(self):
"""Start the daemon's server."""
self.log.info('NICOS daemon v%s started, starting server on %s',
nicos_version, self.server)
# startup the script thread
self._controller.start_script_thread()
self._worker = createThread('daemon server', self._server.start,
args=(self._long_loop_delay,))
def doStart(self, target):
self._stop_request = False
if self._regulation_thread is None and session.sessiontype != POLLER:
# no regulation thread (yet), but running in daemon -> start one
self._regulation_thread = createThread(
'regulation thread %s' % self, self._regulate)
self.curstatus = status.BUSY, 'regulating'
self.poll()
def doInit(self, mode):
if mode == SIMULATION:
return
# create only run ONE thread: in the poller
# it may look stupid, as the poller already has a thread polling read()
# now imagine several such devices in a setup.... not so stupid anymore
if session.sessiontype == POLLER:
self._thread = createThread('measure cpuload', self._run)
def doInit(self, mode):
if mode == SIMULATION:
return
if self.curstatus[0] < status.OK: # clean up old status values
self._setROParam('curstatus', (status.OK, ''))
if session.sessiontype != POLLER: # dont run in the poller!
self._window = []
self._statusLock = threading.Lock()
self._thread = createThread('cryo simulator %s' % self, self.__run)
def doInit(self, mode):
if requests is None:
raise ConfigurationError(self, 'requests package is missing')
self._prefix = self.prefix.strip('/')
if self._prefix:
self._prefix += '/'
self._initFilters()
self._queue = queue.Queue(1000)
self._processor = createThread('webhookprocessor', self._processQueue)
def doStart(self, pos):
if pos == self.doRead():
return
if pos == 'Ell':
self._attached_collimator.move(0)
self._attached_ellipse.move(1)
elif pos == 'Col':
self._attached_ellipse.move(0)
self._attached_collimator.move(1)
self._timer = False
self._thread = createThread('sleep', self._wait)
def stop(*devlist):
"""Stop one or more devices.
If no device is given, stop all stoppable devices in parallel.
Examples:
>>> stop(phi) # stop the phi device
>>> stop(phi, psi) # stop the phi and psi devices
>>> stop() # stop all devices
"""
stop_all = False
if not devlist:
stop_all = True
devlist = [
session.devices[devname] for devname in session.explicit_devices
if isinstance(session.devices[devname], (Moveable, Measurable))
]
finished = []
def stopdev(dev):
try:
dev.stop()
if not stop_all:
dev.log.info('stopped')
except AccessError:
# do not warn about devices we cannot access if they were not
# explicitly selected
pass
except Exception:
dev.log.warning('error while stopping', exc=1)
finally:
finished.append(dev)
for dev in devlist:
dev = session.getDevice(dev)
createThread('device stopper %s' % dev, stopdev, (dev, ))
while len(finished) != len(devlist):
session.delay(Device._base_loop_delay)
if stop_all:
session.log.info('all devices stopped')
def addPollDev(self, device):
"""adds a device to our simplified polling loop
also starts the polling thread, if necessary
"""
if self._polls:
if device not in self._polls:
self._polls.append(device)
else:
self._polls = [device]
if (self._thread is None) or not(self._thread.is_alive()):
self._thread = createThread('servo poller', self._poller)
def start(self, interval):
createThread('daemon event sender', self.event_sender)
# TODO:
# * clean up unused handlers (when?)
# * more zmq inproc sockets and proxies instead of queues?
# * useful and comprehensive error handling
reply_collect = nicos_zmq_ctx.socket(zmq.PULL)
reply_collect.bind('inproc://daemon_reply')
poller = zmq.Poller()
poller.register(self.sock, zmq.POLLIN)
poller.register(reply_collect, zmq.POLLIN)
# ZeroMQ expects a poll timeout given in msec. interval is passed
# through in seconds.
interval_ms = interval * 1000
while not self._stoprequest:
for (sock, _) in poller.poll(interval_ms):
# reply? pass it through
if sock is reply_collect:
self.sock.send_multipart(reply_collect.recv_multipart())
continue
# otherwise, must be message from a client
msg = self.sock.recv_multipart()
client_id = msg[0]
if client_id in self.handlers:
self.handlers[client_id].command_queue.put(msg)
elif msg[2] == b'getbanner':
# new connection, create a handler
self.handler_ident += 1
handler = ServerTransport(client_id, self)
with self.handler_lock:
self.handlers[client_id] = handler
createThread('handler %d' % self.handler_ident,
handler.handle_loop)
else:
# all other messages: client must initiate connection first
self.sock.send_multipart([client_id, b'', b'error', b'',
b'"session expired"'])
def doInit(self, mode):
BaseCacheClient.doInit(self, mode)
# cache of all interesting keys with current values
self._keydict = LCDict()
# put status constants in key dict to simplify status conditions
for stval, stname in status.statuses.items():
self._keydict[stname.upper()] = stval
# set to true during connect action
self._process_updates = False
# current setups
self._setups = set()
# mapping entry ids to entrys
self._entries = {}
# (mangled) key to update mail receivers
self._mailreceiverkey = self.mailreceiverkey.replace('/', '_').lower()
# mapping cache keys to entries that check this key
self._keymap = {'session_mastersetup': set()}
if self._mailreceiverkey:
self._keymap[self._mailreceiverkey] = set()
# current warnings: mapping entry ids to the string description
self._warnings = OrderedDict()
# current count loop pause reasons: mapping like self._warnings
self._pausecount = OrderedDict()
# create all notifier devices
self._all_notifiers = []
self._notifiers = {'': []}
for key, devnames in self.notifiers.items():
self._notifiers[key] = notiflist = []
for devname in devnames:
dev = session.getDevice(devname, Notifier)
notiflist.append(dev)
self._all_notifiers.append(dev)
# process entries in the default watchlist
for entry_dict in self.watch:
self._add_entry(entry_dict, 'watchdog')
# start a thread checking for modification of the setup file
createThread('refresh checker', self._checker)
def inner_count(detectors, preset, temporary=False, threaded=False):
"""Inner counting function for normal counts with single-point dataset.
If *temporary* is true, use a dataset without data sinks.
If *threaded* is true, do a non-blocking acquisition.
"""
# stop previous inner_count / acquisition thread if available
stop_acquire_thread()
dataman = session.experiment.data
if session.experiment.forcescandata:
dataman.beginScan(info=preset.get(
'info',
'count(%s)' % ', '.join('%s=%s' % kv for kv in preset.items())),
npoints=1,
environment=session.experiment.sampleenv,
detectors=detectors,
preset=preset)
# start counting
args = dict(detectors=detectors,
environment=session.experiment.sampleenv,
preset=preset)
if temporary:
point = dataman.beginTemporaryPoint(**args)
else:
point = dataman.beginPoint(**args)
read_environment(session.experiment.sampleenv)
def _acquire_func():
try:
acquire(point, preset)
finally:
dataman.finishPoint()
if session.experiment.forcescandata:
dataman.finishScan()
if threaded:
session._thd_acquire = createThread("acquire", _acquire_func)
return None
else:
_acquire_func()
result, msg = CountResult.from_point(point)
if not session.experiment.forcescandata:
for filename in point.filenames:
msg.append('file = %s' % filename)
session.log.info('count: %s', ', '.join(msg))
return result
def doStart(self, target):
if self._mode == SIMULATION:
for p in self.range:
self._attached_moveable._sim_setValue(p)
return
if target == 'on':
self._stop()
self._stop_request = False
if not self._osc_thread:
self._osc_thread = createThread('oscillation thread %s' % self,
self.__oscillation)
else:
self._stop()
self.poll()
def doInit(self, mode):
self._dev_to_value_cache = {}
self._dev_to_status_cache = {}
self._dev_to_timestamp_cache = {}
self._producer = None
self._lock = Lock()
self._initFilters()
self._queue = queue.Queue(1000)
self._worker = createThread('cache_to_kafka', self._processQueue,
start=False)
self._regular_update_worker = createThread(
'send_regular_updates', self._poll_updates, start=False)
while not self._producer:
try:
self._producer = \
KafkaProducer(bootstrap_servers=self._config['brokers'])
except Exception as error:
self.log.error(
'Could not connect to Kafka - will try again soon: %s',
error)
time.sleep(5)
self.log.info('Connected to Kafka brokers %s', self._config['brokers'])
def verify_request(self, request, client_address):
"""Called on a new connection. If we have a connection from that
host that lacks an event connection, use this.
"""
host = client_address[0]
clid = request.recv(16)
if (host, clid) not in self.pending_clients:
self.pending_clients[host, clid] = None
return clid
# this should be an event connection: start event sender thread on the
# handler, but wait until the handler is registered
while self.pending_clients[host, clid] is None:
time.sleep(0.2)
handler = self.pending_clients[host, clid]
self.daemon.log.debug('event connection from %s for handler #%d',
host, handler.ident)
handler.event_sock = request
# close connection after socket send queue is full for 60 seconds
handler.event_sock.settimeout(60.0)
createThread('event_sender %d' % handler.ident, handler.event_sender)
self.pending_clients.pop((host, clid), None)
# don't call the usual handler
return None
def doInit(self, mode):
Experiment.doInit(self, mode)
self._client = None
self._update_cache_worker = createThread('update_cache',
self._update_cache,
start=False)
# Get secret from the environment
token = os.environ.get('YUOS_TOKEN')
if token:
try:
self._client = YuosClient(self.server_url, token,
self.instrument, self.cache_filepath)
self._update_cache_worker.start()
except BaseYuosException as error:
self.log.warn(f'QueryDB not available: {error}')
def _startPositioningThread(self, target):
if self._posthread:
if self._posthread.is_alive():
self._posthread.join()
self._posthread = None
self._stoprequest = 0
self._errorstate = None
if self._checkTargetPosition(self.read(0), target, error=False):
self.log.debug('not moving, already at %.4f within precision',
target)
return
self._target = target
self._posthread = createThread('positioning thread %s' % self,
self.__positioningThread)
def doStart(self, pos):
if self.curstatus[0] == status.DISABLED:
raise MoveError(self, 'cannot move, motor is disabled')
pos = float(pos) + self.offset
if self._thread:
self._setROParam('curstatus', (status.BUSY, 'waiting for stop'))
self._stop = True
self._thread.join()
if self.speed != 0:
self._setROParam('curstatus', (status.BUSY, 'virtual moving'))
self._thread = createThread('virtual motor %s' % self,
self.__moving, (pos, ))
else:
self.log.debug('moving to %s', pos)
self._setROParam('curvalue', pos)
self._setROParam('curstatus', (status.OK, 'idle'))
def multiReference(dev, subdevs, parallel=False):
"""Run a reference drive of *subdevs* (belonging to the main *dev*).
If *parallel* is true, use one thread per device.
"""
from nicos.devices.abstract import CanReference
if not parallel:
for subdev in subdevs:
if isinstance(subdev, CanReference):
dev.log.info('referencing %s...', subdev)
subdev.reference()
else:
dev.log.warning('%s cannot be referenced', subdev)
return
def threaded_ref(i, d):
try:
d.reference()
except Exception:
d.log.error('while referencing', exc=1)
errored[i] = True
threads = []
errored = [False] * len(subdevs)
for (i, subdev) in enumerate(subdevs):
if isinstance(subdev, CanReference):
dev.log.info('referencing %s...', subdev)
threads.append(
createThread('reference %s' % subdev, threaded_ref,
(i, subdev)))
else:
dev.log.warning('%s cannot be referenced', subdev)
for thread in threads:
thread.join()
if any(errored):
raise MoveError(
dev, 'referencing failed for ' + ', '.join(
str(subdev) for (subdev, err) in zip(subdevs, errored) if err))
