def _connect_action(self): @timedRetryOnExcept(max_retries=1, timeout=self._locktimeout, ex=CacheLockError) def trylock(): return self.lock('elog') try: trylock() except CacheLockError as err: self.log.info('another elog is already active: %s', sessionInfo(err.locked_by)) sys.exit(-1) else: self._islocked = True # request current directory for the handler to start up correctly self._socket.sendall( to_utf8('logbook/directory%s\n###%s\n' % (OP_ASK, OP_ASK))) # read response data, n = b'', 0 while not data.endswith(b'###!\n') and n < 1000: data += self._socket.recv(8192) n += 1 self.storeSysInfo('elog') # send request for all relevant updates self._socket.sendall(to_utf8('@logbook/%s\n' % OP_SUBSCRIBE)) self._process_data(data)
def _connect_action(self): self._keys_expired = False BaseCacheClient._connect_action(self) if self.showwatchdog: # also ask for and subscribe to all watchdog events self._socket.sendall(to_utf8('@watchdog/%s\n' % OP_WILDCARD)) self._socket.sendall(to_utf8('@watchdog/%s\n' % OP_SUBSCRIBE)) # use appname to distinguish between different instances self.storeSysInfo(session.appname)
def emit(self, html, suffix=u''): html = to_utf8(html) suffix = to_utf8(suffix) if self.fd: self.fd.write(html) # write suffix now, but place file pointer so that it's overwritten # on subsequent writes in the same state -- this way we can # guarantee that tags don't stay open if suffix: self.fd.write(suffix) self.fd.flush() self.fd.seek(-len(suffix), 2) else: self.fd.flush()
def _write_instrument(self, valuelist): # self.log.info('Instrument: %r', valuelist) instrument = self._data['instrument'] for device, key, value in valuelist: if device not in ['demo', 'DEMO']: if key in ['facility', 'website']: instrument[key] = to_utf8(value) elif key == 'instrument': instrument['name'] = value elif key == 'operators': instrument[key] = [] for operator in value: instrument[key].append(to_utf8(operator)) elif key == 'doi': instrument['references'] = [] instrument['references'].append(to_utf8(value))
def encode(key, entry): # Start with a buffer which can automatically grow builder = flatbuffers.Builder(136) # Create the strings - this has to be done before starting to build value_fb_str = None if entry.value is not None: value_fb_str = builder.CreateString(entry.value) key_fb_str = builder.CreateString(key) # Start building the buffer. # Flatbuffer must be constructed in the reverse order of the schema. # This might be a bug in flatbuffers. CacheEntryFB.CacheEntryStart(builder) if value_fb_str: CacheEntryFB.CacheEntryAddValue(builder, value_fb_str) CacheEntryFB.CacheEntryAddExpired(builder, entry.expired) # Do not write ttl if it is None if entry.ttl is not None: CacheEntryFB.CacheEntryAddTtl(builder, entry.ttl) CacheEntryFB.CacheEntryAddTime(builder, entry.time) CacheEntryFB.CacheEntryAddKey(builder, key_fb_str) fb_entry = CacheEntryFB.CacheEntryEnd(builder) builder.Finish(fb_entry, file_identifier=to_utf8(file_identifier)) # Generate the output and replace the file_identifier fb_array = builder.Output() return bytes(fb_array)
def add_legacy(self, pv_details): """ Configure given PVs to start forwarding. Specific topics and schemas can be provided. If not, default instrument topic and schemas will be used. :param pv_details: PVs and the tuple of (topic, schema) :type pv_details: dict(pvname, (kafka-topic, schema)) """ streams = [] for pv in pv_details: topic, schema = pv_details[pv] if not topic: topic = self.instpvtopic if not schema: schema = self.instpvschema self._issued[pv] = (topic, schema) for broker in self.brokers: converter = { "topic": f"{broker}/{topic}", "schema": schema, } stream = { "converter": converter, "channel_provider_type": "ca", "channel": pv, } streams.append(stream) cmd = {"cmd": "add", "streams": streams} self.send(self.cmdtopic, to_utf8(json.dumps(cmd)))
def _hash(self, password): password = to_utf8(from_maybe_utf8(password)) if self.hashing == 'sha1': password = hashlib.sha1(password).hexdigest() elif self.hashing == 'md5': password = hashlib.md5(password).hexdigest() return password
def _sender_thread(self): while not self.stoprequest: data = self.send_queue.get() # self.log.debug('sending: %r', data) if self.sock is None: # connection already closed return while True: try: self.sock.sendall(to_utf8(data)) except socket.timeout: self.log.warning('send timed out, shutting down') self.closedown() except socket.error as err: if err.args[0] == EAGAIN: sleep(CYCLETIME) continue self.log.warning('other end closed, shutting down', exc=err) self.closedown() except Exception: # if we can't write (or it would be blocking), there is some # serious problem: forget writing and close down self.log.warning('other end closed, shutting down') self.closedown() break
def _process_data(self, data, sync_str=to_utf8(SYNC_MARKER + OP_TELLOLD), lmatch=line_pattern.match, mmatch=msg_pattern.match): # n = 0 i = 0 # avoid making a string copy for every line match = lmatch(data, i) while match: line = match.group(1) i = match.end() if sync_str in line: self.log.debug('process data: received sync: %r', line) self._synced = True else: msgmatch = mmatch(from_utf8(line)) # ignore invalid lines if msgmatch: # n += 1 try: self._handle_msg(**msgmatch.groupdict()) except Exception: self.log.exception('error handling message %r', msgmatch.group()) # continue loop match = lmatch(data, i) # self.log.debug('processed %d items', n) return data[i:]
def _receiver_thread(self): # we will never read any more data: just process what we got and send # any needed responses synchronously try: self._process_data(self.data, lambda reply: self._sendall(to_utf8(reply))) except Exception as err: self.log.warning('error handling UDP data %r', self.data, exc=err) self.closedown()
def ListAuth(request): passwds = [] for (user, pw, level) in request.function.passwd: # note: we currently allow empty password to match any password! # pylint: disable=compare-to-empty-string hashed = hashlib.sha1(to_utf8(pw)).hexdigest() if pw != '' else pw passwds.append((user, hashed, level)) Auth = ListAuthenticator('authenicator', hashing='sha1', passwd=passwds) yield Auth
def storeSysInfo(self, service): """Store info about the service in the cache.""" if not self._socket: return try: key, res = getSysInfo(service) msg = '%s@%s%s%s\n' % (currenttime(), key, OP_TELL, cache_dump(res)) self._socket.sendall(to_utf8(msg)) except Exception: self.log.exception('storing sysinfo failed')
def _writeLogs(self): if not self._logfile: return loglines = [] loglines.append('%-15s\tmean\tstdev\tmin\tmax' % '# dev') for dev in self.dataset.valuestats: loglines.append('%-15s\t%.3f\t%.3f\t%.3f\t%.3f' % ((dev, ) + self.dataset.valuestats[dev])) self._logfile.seek(0) for line in loglines: self._logfile.write(to_utf8('%s\n' % line)) self._logfile.flush()
def on_saveBtn_clicked(self): initialdir = self.client.eval('session.experiment.proposalpath', '') fn = QFileDialog.getSaveFileName(self, 'Save protocol', initialdir, 'Text files (*.txt)')[0] if not fn: return try: text = self.outText.toPlainText() with open(fn, 'wb') as fp: fp.write(to_utf8(text)) except Exception as err: self.showError('Could not save: %s' % err)
def _write_exptype(self, l): data = 'EXPTYPE' self._defcmd(data) self._string(data) buf = b'\x80' sbuf = b'' for item in l: item = to_utf8(item) buf += pack('BBBB', 0x80, CHARTYPE, 0x81, len(item)) sbuf += item buf += b'\x81\x01' self._file_write(buf + sbuf)
def mainLoop(self): while not self._stoprequest: try: if self._content: content = u''.join(ct.getHTML() for ct in self._content) safeWriteFile(self.filename, to_utf8(content), mode='wb', maxbackups=0) except Exception: self.log.error('could not write status to %r', self.filename, exc=1) else: self.log.debug('wrote status to %r', self.filename) sleep(self.interval)
def doFinish(self): """Automatic protocol generation before finishing a user experiment.""" if self.proptype != 'user': return proto_path = path.join(self.proposalpath, 'protocol.txt') try: text = self._generate_protocol(with_ts=True) with open(proto_path, 'wb') as fp: fp.write(to_utf8(text)) except Exception: self.log.warning('Error during protocol generation', exc=1) else: self.log.info('Protocol generated at %s', proto_path)
def _connect_action(self): # send request for all keys and updates.... # (send a single request for a nonexisting key afterwards to # determine the end of data) msg = '@%s%s\n%s%s\n' % (self._prefix, OP_WILDCARD, END_MARKER, OP_ASK) self._socket.sendall(to_utf8(msg)) # read response data, n = b'', 0 sentinel = to_utf8(END_MARKER + OP_TELLOLD + '\n') while not data.endswith(sentinel) and n < 1000: data += self._socket.recv(BUFSIZE) n += 1 # send request for all updates msg = '@%s%s\n' % (self._prefix, OP_SUBSCRIBE) self._socket.sendall(to_utf8(msg)) for prefix in self._prefixcallbacks: msg = '@%s%s\n' % (prefix, OP_SUBSCRIBE) self._socket.sendall(to_utf8(msg)) self._process_data(data)
def _connect_action(self): # like for BaseCacheClient, but without request for updates msg = '@%s%s\n###%s\n' % (self._prefix, OP_WILDCARD, OP_ASK) self._socket.sendall(to_utf8(msg)) # read response data, n = b'', 0 while not data.endswith(b'###!\n') and n < 1000: data += self._socket.recv(BUFSIZE) n += 1 self._process_data(data) # stop immediately after reading data self._stoprequest = True
def authenticate(self, username, password): try: pam.authenticate(username, password, resetcred=0) entry = pwd.getpwnam(username) idx = access_re.search(to_utf8(entry.pw_gecos)) if idx: access = int(idx.group('level')) if access in (GUEST, USER, ADMIN): return User(username, access) return User(username, self.defaultlevel) except pam.PAMError as err: raise AuthenticationError('PAM authentication failed: %s' % err) except Exception as err: raise AuthenticationError( 'exception during PAM authentication: %s' % err)
def writeData(self, fp, image): _metainfo = self.dataset.metainfo detectors = self.sink.detectors detector = detectors[0] if detectors else 'adet' _resosteps = _metainfo[detector, 'resosteps'][0] _range = _metainfo[detector, 'range'][0] _stepsize = _range / _resosteps _startpos = _metainfo[detector, '_startpos'][0] _start = _startpos - (_resosteps - 1) * _stepsize fp.write(b'position\tcounts\n') for i, v in enumerate(image): _pos = _start + i * _stepsize fp.write(to_utf8('%.2f\t%d\n' % (_pos, v.sum()))) fp.flush()
def _update_topic(self, key, entry): # This method is responsible to communicate and update all the # topics that should be updated. Subclasses can (re)implement it # if there are messages to be produced to other topics self.log.debug('writing: %s -> %s', key, entry.value) # For the log-compacted topic key deletion happens when None is # passed as the value for the key value = None if entry.value is not None: # Only when the key deletion is not required value = self._attached_serializer.encode(key, entry) self._producer.send(topic=self.currenttopic, value=value, key=to_utf8(key), timestamp_ms=int(entry.time * 1000)) # clear all local buffers and produce pending messages self._producer.flush()
def txtplot(x, y, xlab, ylab, xterm_mode=False): """Plot data with gnuplot's dumb ASCII terminal.""" if not x.size: raise ValueError('Empty plot') if len(x) != len(y): raise ValueError('Unequal lengths of X and Y values') try: gnuplot = createSubprocess(['gnuplot', '--persist'], shell=False, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if xterm_mode: cmd = ['set term xterm'] else: cmd = ['set term dumb'] cmd.append('set xlabel "' + xlab + '"') cmd.append('set ylabel "' + ylab + '"') cmd.append('plot "-" with points notitle') for xy in zip(x, y): cmd.append('%s %s' % xy) cmd.append('e\n') cmd = '\n'.join(cmd) out = gnuplot.communicate(to_utf8(cmd))[0] lines = [line for line in out.splitlines() if line] if xterm_mode: lines += ['Plotting in xterm Tektronix window.', '\x1b_If you can only see a lot of incomprehensible ' 'text, use xterm instead of your current terminal ' 'emulator.\x1b\\'] return lines except OSError: raise RuntimeError('Could execute gnuplot for text plot')
def ftpUpload(filename, logger=None): """Uploads the given file to an user-accessible location returns a http download link for download purposes. """ # we like to obscure the data at least a little bit. subdir = md5(to_utf8(filename + str(time.time()))).hexdigest() basename = path.basename(filename) try: with open(filename, 'rb') as fp: ftp = FTP() ftp.connect(FTP_SERVER, FTP_PORT) ftp.login(FTP_USER, FTP_P) try: ftp.mkd( subdir) # may raise if dir already exists. Should be rare! except Exception: pass ftp.cwd(subdir) ftp.storbinary('STOR %s' % basename, fp) ftp.quit() ftp.close() except Exception: if logger: logger.error( 'Uploading ftp-file failed! Please check config and ' 'log files', exc=1) raise return 'http://ftp.frm2.tum.de/outgoing/mdata/%s/%s' % (subdir, basename)
def _single_request(self, tosend, sentinel=b'\n', retry=2, sync=False): """Communicate over the secondary socket.""" if not self._socket: self._disconnect('single request: no socket') if not self._socket: raise CacheError('cache not connected') if sync: # sync has to be false for lock requests, as these occur during startup self._queue.join() with self._sec_lock: if not self._secsocket: try: self._secsocket = tcpSocket(self.cache, DEFAULT_CACHE_PORT) except Exception as err: self.log.warning( 'unable to connect secondary socket ' 'to %s: %s', self.cache, err) self._secsocket = None self._disconnect('secondary socket: could not connect') raise CacheError('secondary socket could not be created') try: # write request # self.log.debug("get_explicit: sending %r", tosend) self._secsocket.sendall(to_utf8(tosend)) # give 10 seconds time to get the whole reply timeout = currenttime() + 10 # read response data = b'' while not data.endswith(sentinel): newdata = self._secsocket.recv(BUFSIZE) # blocking read if not newdata: raise socket.error('cache closed connection') if currenttime() > timeout: # do not just break, we need to reopen the socket raise socket.error('getting response took too long') data += newdata except socket.error: self.log.warning('error during cache query', exc=1) closeSocket(self._secsocket) self._secsocket = None if retry: for m in self._single_request(tosend, sentinel, retry - 1): yield m return raise lmatch = line_pattern.match mmatch = msg_pattern.match i = 0 # self.log.debug("get_explicit: data =%r", data) match = lmatch(data, i) while match: line = match.group(1) i = match.end() msgmatch = mmatch(from_utf8(line)) if not msgmatch: # ignore invalid lines continue # self.log.debug('line processed: %r', line) yield msgmatch match = lmatch(data, i)
def _worker_inner(self): data = b'' process = self._process_data while not self._stoprequest: if self._should_connect: if not self._socket: self._connect() if not self._socket: self._wait_retry() continue else: if self._socket: self._disconnect() self._wait_retry() continue # process data so far data = process(data) # wait for a whole line of data to arrive while b'\n' not in data and self._socket and self._should_connect \ and not self._stoprequest: # optionally do some action while waiting self._wait_data() if self._queue.empty(): # NOTE: the queue.empty() check is not 100% reliable, but # that is not important here: all we care is about not # having the select always return immediately for writing writelist = [] else: writelist = [self._socket] # read or write some data while 1: try: res = select.select([self._socket], writelist, [], self._selecttimeout) except EnvironmentError as e: if e.errno == errno.EINTR: continue raise except TypeError: # socket was None, let the outer loop handle that res = ([], [], []) break if res[1]: # determine if something needs to be sent tosend = '' itemcount = 0 try: # bunch a few messages together, but not unlimited for _ in xrange(10): tosend += self._queue.get(False) itemcount += 1 except queue.Empty: pass # write data try: self._socket.sendall(to_utf8(tosend)) except Exception: self._disconnect('disconnect: send failed') # report data as processed, but then re-queue it to send # after reconnect for _ in range(itemcount): self._queue.task_done() data = b'' self._queue.put(tosend) break for _ in range(itemcount): self._queue.task_done() if res[0]: # got some data try: newdata = self._socket.recv(BUFSIZE) except Exception: newdata = b'' if not newdata: # no new data from blocking read -> abort self._disconnect('disconnect: recv failed') data = b'' break data += newdata if self._socket: # send rest of data tosend = '' itemcount = 0 try: while 1: tosend += self._queue.get(False) itemcount += 1 except queue.Empty: pass try: self._socket.sendall(to_utf8(tosend)) except Exception: self.log.debug('exception while sending last batch of updates', exc=1) # no reraise, we'll disconnect below anyways for _ in range(itemcount): self._queue.task_done() # end of while loop self._disconnect()
def writeHeader(self, fp, metainfo, image): shape = image.shape try: SD = '%.4f' % ((session.getDevice('det1_z').read() - session.getDevice('st1_x').read()) / 1000) except Exception: self.log.warning( "can't determine SD (detector distance), " "using 0 instead", exc=1) SD = 0 finished = currenttime() # totalTime = finished - self.dataset.started Sum = image.sum() Moni1 = 0 Moni2 = 0 Time = 0 try: Moni1 = float(session.getDevice('det1_mon1').read()[0]) Moni2 = float(session.getDevice('det1_mon2').read()[0]) Time = float(session.getDevice('det1_timer').read()[0]) except Exception: self.log.warning( "can't determine all monitors, " "using 0.0 instead", exc=1) try: # Setupfile = session.getDevice('det1_image').histogramfile Histfile = metainfo['det1_image', 'histogramfile'][1] except Exception: Histfile = '' try: # Listfile = session.getDevice('det1_image').listmodefile.split('\'')[1] Listfile = metainfo['det1_image', 'listmodefile'][1].split('\'')[1] except Exception: Listfile = '' try: # Setupfile = session.getDevice('det1_image').configfile Setupfile = metainfo['det1_image', 'configfile'][1] except Exception: Setupfile = 'setup' try: # LookUpTable = session.getDevice('det1_image').calibrationfile LookUpTable = metainfo['det1_image', 'calibrationfile'][1] except Exception: LookUpTable = 'lookup' time_format = '%I:%M:%S %p' date_format = '%m/%d/%Y' metadata = DeviceValueDict( fileName=os.path.basename(self._file.filepath), fileDate=strftime(date_format, localtime(self.dataset.started)), fileTime=strftime(time_format, localtime(self.dataset.started)), FromDate=strftime(date_format, localtime(self.dataset.started)), FromTime=strftime(time_format, localtime(self.dataset.started)), ToDate=strftime(date_format, localtime(finished)), ToTime=strftime(time_format, localtime(finished)), DataSize=shape[0] * shape[1], DataSizeX=shape[1], DataSizeY=shape[0], Environment='_'.join(session.explicit_setups), SD=SD, Sum='%d' % Sum, Moni1='%d' % Moni1, Moni2='%d' % Moni2, Sum_Time='%.6f' % (Sum / Time) if Time else 'Inf', Sum_Moni1='%.6f' % (Sum / Moni1) if Moni1 else 'Inf', Sum_Moni2='%.6f' % (Sum / Moni2) if Moni2 else 'Inf', Histfile=Histfile, Listfile=Listfile, Setupfile=Setupfile, LookUpTable=LookUpTable, Command=self.dataset.info, ) nicosheader = [] # no way to map nicos-categories to BerSANS sections :( # also ignore some keys :( ignore = ('det1_lastlistfile', 'det1_lasthistfile') for (dev, param), (value, strvalue, _unit, _category) in \ iteritems(self.dataset.metainfo): devname_key = '%s_%s' % (dev, param) if devname_key in ignore: continue metadata[devname_key] = value nicosheader.append('%s=%s' % (devname_key, strvalue)) nicosheader = b'\n'.join(sorted(map(to_ascii_escaped, nicosheader))) self.log.debug('nicosheader starts with: %40s', nicosheader) # write Header header = BERSANSHEADER if 'tisane' in session.explicit_setups: header += TISANEHEADER for line in header.split('\n'): self.log.debug('testing header line: %r', line) self.log.debug(line % metadata) fp.write(to_utf8(line % metadata)) fp.write(b'\n') # also append nicos header fp.write(nicosheader.replace(b'\\n', b'\n')) # why needed? fp.write(b'\n\n%Counts\n') fp.flush()
def connect(self, conndata, eventmask=None): """Connect to a NICOS daemon. *conndata* is a ConnectionData object. *eventmask* is a tuple of event names that should not be sent to this client. """ self.disconnecting = False if self.isconnected: raise RuntimeError('client already connected') try: self.transport.connect(conndata) except socket.error as err: msg = err.args[1] if len(err.args) >= 2 else str(err) self.signal('failed', 'Server connection failed: %s.' % msg, err) return except Exception as err: self.signal('failed', 'Server connection failed: %s.' % err, err) return # read banner try: success, banner = self.transport.recv_reply() if not success: raise ProtocolError('invalid response format') if 'daemon_version' not in banner: raise ProtocolError('daemon version missing from response') daemon_proto = banner.get('protocol_version', 0) if daemon_proto != PROTO_VERSION: if daemon_proto in COMPATIBLE_PROTO_VERSIONS: self.compat_proto = daemon_proto else: raise ProtocolError('daemon uses protocol %d, but this ' 'client requires protocol %d, do you ' 'need to update NICOS?' % (daemon_proto, PROTO_VERSION)) except Exception as err: self.signal( 'failed', 'Server (%s:%d) handshake failed: %s.' % (conndata.host, conndata.port, err), err) return # log-in sequence self.isconnected = True password = conndata.password pw_hashing = banner.get('pw_hashing', 'sha1') if pw_hashing[0:4] == 'rsa,': if rsa is not None: encodedkey = banner.get('rsakey', None) if encodedkey is None: raise ProtocolError( 'rsa requested, but rsakey missing in banner') if not PY2 and not isinstance(encodedkey, bytes): encodedkey = bytes(encodedkey, 'utf-8') pubkey = rsa.PublicKey.load_pkcs1(b64decode(encodedkey)) password = rsa.encrypt(to_utf8(password), pubkey) password = '******' + b64encode(password).decode() else: pw_hashing = pw_hashing[4:] if pw_hashing == 'sha1': password = hashlib.sha1(to_utf8(password)).hexdigest() elif pw_hashing == 'md5': password = hashlib.md5(to_utf8(password)).hexdigest() credentials = { 'login': conndata.user, 'passwd': password, 'display': '', } response = self.ask('authenticate', credentials) if not response: self._close() return self.user_level = response['user_level'] if eventmask: self.tell('eventmask', eventmask) self.transport.connect_events(conndata) # start event handler self.event_thread = createThread('event handler', self.event_handler) self.host, self.port = conndata.host, conndata.port self.login = conndata.user self.viewonly = conndata.viewonly self.daemon_info = banner self.signal('connected')
def send_command(self, cmdname, args): data = self.serializer.serialize_cmd(cmdname, args) self.sock.send_multipart([to_utf8(cmdname), '', data])
def _string(self, value): value = to_utf8(value) buf = pack('B', CHARTYPE) + self._len(len(value)) + value.upper() self._file_write(buf)