def reset_signal_handler(cls, signal_handler): """ Class state: - Overwrites `cls._signal_handler`. OS state: - Overwrites signal handlers for SIGINT, SIGQUIT, and SIGTERM. NB: This method calls signal.signal(), which will crash if not called from the main thread! :returns: The :class:`SignalHandler` that was previously registered, or None if this is the first time this method was called. """ assert(isinstance(signal_handler, SignalHandler)) # NB: Modify process-global state! for signum, handler in signal_handler.signal_handler_mapping.items(): signal.signal(signum, handler) # Retry any system calls interrupted by any of the signals we just installed handlers for # (instead of having them raise EINTR). siginterrupt(3) says this is the default behavior on # Linux and OSX. signal.siginterrupt(signum, False) previous_signal_handler = cls._signal_handler # NB: Mutate the class variables! cls._signal_handler = signal_handler return previous_signal_handler
def add_signal_handler(self, sig, callback, *args): """Add a handler for a signal. UNIX only. Raise ValueError if the signal number is invalid or uncatchable. Raise RuntimeError if there is a problem setting up the handler. """ self._check_signal(sig) try: # set_wakeup_fd() raises ValueError if this is not the # main thread. By calling it early we ensure that an # event loop running in another thread cannot add a signal # handler. signal.set_wakeup_fd(self._csock.fileno()) except ValueError as exc: raise RuntimeError(str(exc)) handle = events.make_handle(callback, args) self._signal_handlers[sig] = handle try: signal.signal(sig, self._handle_signal) # Set SA_RESTART to limit EINTR occurrences. signal.siginterrupt(sig, False) except OSError as exc: del self._signal_handlers[sig] if not self._signal_handlers: try: signal.set_wakeup_fd(-1) except ValueError as nexc: logger.info('set_wakeup_fd(-1) failed: %s', nexc) if exc.errno == errno.EINVAL: raise RuntimeError('sig {} cannot be caught'.format(sig)) else: raise
def daemonize(do_fork=True, skip_fds=[]): """Daemonizes current process.""" if do_fork: if os.fork(): os._exit(0) else: os.setsid() if os.fork(): os._exit(0) os.chdir("/") os.umask(0) signal.signal(signal.SIGHUP, signal.SIG_IGN) signal.siginterrupt(signal.SIGHUP, False) # Redirecting standard streams to /dev/null and closing original descriptors null_dev = eintr_retry(os.open)("/dev/null", os.O_RDWR) try: for fd in (sys.stdin.fileno(), sys.stdout.fileno(), sys.stderr.fileno()): if fd not in skip_fds: os.dup2(null_dev, fd) finally: eintr_retry(os.close)(null_dev)
def main(): signal.siginterrupt(signal.SIGINT, True) signal.signal(signal.SIGINT, exit) args = sys.argv if len(args) != 2 or args[1][-4:] == 'help': print("usage: python flashcard.py filename.json") sys.exit(0) with open(args[1], 'r') as f: text = f.read() d = json.loads(text) keys = list(d.keys()) n = len(keys) while True: os.system('clear') print("Starting a new round") print("Press enter to proceed") input() for i in range(n): target = random.randrange(i, n) keys[i], keys[target] = keys[target], keys[i] koi = keys[i] os.system('clear') print(koi) input() print(d[koi]) input()
def init_signals(self): # Set up signals through the event loop API. self.loop.add_signal_handler(signal.SIGQUIT, self.handle_quit, signal.SIGQUIT, None) self.loop.add_signal_handler(signal.SIGTERM, self.handle_exit, signal.SIGTERM, None) self.loop.add_signal_handler(signal.SIGINT, self.handle_quit, signal.SIGINT, None) self.loop.add_signal_handler(signal.SIGWINCH, self.handle_winch, signal.SIGWINCH, None) self.loop.add_signal_handler(signal.SIGUSR1, self.handle_usr1, signal.SIGUSR1, None) self.loop.add_signal_handler(signal.SIGABRT, self.handle_abort, signal.SIGABRT, None) # Don't let SIGTERM and SIGUSR1 disturb active requests # by interrupting system calls signal.siginterrupt(signal.SIGTERM, False) signal.siginterrupt(signal.SIGUSR1, False)
def server_config(): global log_file, log_buffer global data_fd, data_fifo #SIGINT clean up and quit signal.signal(signal.SIGINT, sigint_handler) #keep alive timer signal.signal(signal.SIGALRM, sigtimer_handler) signal.setitimer(signal.ITIMER_REAL, KEEP_ALIVE, KEEP_ALIVE) #syscall interrupt: restart signal.siginterrupt(signal.SIGALRM, False) signal.siginterrupt(signal.SIGUSR1, False) #server status signal.signal(signal.SIGUSR1, sigusr1_handler) #data channel data_fifo = "nonblocking_pid_%d.fifo" % os.getpid() os.mkfifo(data_fifo , 0666) f = open(data_fifo, "r+") data_fd = os.dup(f.fileno()) f.close() #main log if os.path.exists(NONBLOCKING_LOG): log_file = open(NONBLOCKING_LOG, "a+") else: log_file = open(NONBLOCKING_LOG, "w") if LOG_BUFFER_SIZE: log_buffer = StringIO()
def stop(self, signum=None, frame_unused=None): """Stop the consumer from consuming by calling BasicCancel and setting our state. """ LOGGER.debug('Stop called in state: %s', self.state_description) if self.is_stopped: LOGGER.warning('Stop requested but consumer is already stopped') return elif self.is_shutting_down: LOGGER.warning('Stop requested, consumer is already shutting down') return elif self.is_waiting_to_shutdown: LOGGER.warning('Stop requested but already waiting to shut down') return # Stop consuming self.cancel_consumer_with_rabbitmq() # Wait until the consumer has finished processing to shutdown if self.is_processing: LOGGER.info('Waiting for consumer to finish processing') self.set_state(self.STATE_STOP_REQUESTED) if signum == signal.SIGTERM: signal.siginterrupt(signal.SIGTERM, False) return self.on_ready_to_stop()
def main(): pipe_r, pipe_w = os.pipe() flags = fcntl.fcntl(pipe_w, fcntl.F_GETFL, 0) flags = flags | os.O_NONBLOCK fcntl.fcntl(pipe_w, fcntl.F_SETFL, flags) signal.signal(signal.SIGCHLD, lambda x,y: None) signal.signal(signal.SIGALRM, lambda x,y: None) signal.siginterrupt(signal.SIGCHLD,False) #makes no difference signal.siginterrupt(signal.SIGALRM,False) #makes no difference signal.set_wakeup_fd(pipe_w) signal.setitimer(signal.ITIMER_REAL, 2, 2) poller = select.epoll() poller.register(pipe_r, select.EPOLLIN) poller.register(sys.stdin, select.EPOLLIN) print "Main screen turn on" while True: events=[] try: events = poller.poll() try: for fd, flags in events: ch=os.read(fd, 1) if fd==pipe_r: sys.stdout.write( "We get Signal" ) if fd==sys.stdin.fileno(): sys.stdout.write( ch ) sys.stdout.flush() except IOError as e: print "exception loop" + str(e) except IOError as e: print "exception poll" + str(e)
def fastingest(ns): """ A really fast ingest version, unlike the echoprint server utility, which will eat all of you RAM. :param ns: Namespace object with required config :return: None """ es = EchoprintServer( solr_url=ns.solr, tyrant_address=(ns.tyrant_host, ns.tyrant_port) ) import_date = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") status = 0 start_time = datetime.datetime.utcnow() def signal_handler(signum, frame): diff = datetime.datetime.utcnow() - start_time print '{0}, {1}'.format(diff, status) signal.signal(signal.SIGUSR1, signal_handler) signal.siginterrupt(signal.SIGUSR1, False) with committing(es), open(ns.path) as f: data = ijson.items(f, 'item') for item in data: song = Song.from_echoprint(item, import_date=import_date) if song is not None: # don't commit, the contextmanager (with) takes care of it es.ingest(song, commit=False, check_duplicates=ns.check_duplicates) status += 1
def main(prog, args): listenfd = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) # empty string means INADDR_ANY servaddr = ('', const.SERV_PORT) listenfd.bind(servaddr) listenfd.listen(const.LISTENQ) signal.signal(signal.SIGCHLD, sig_chld) signal.siginterrupt(signal.SIGCHLD, False) while True: # XXX # In Python, socket.accept() is written in C so it will block # the main process from receiving the SIGCHLD signal # Thus the child process will remain in zombie status untill # next connection from client and this accept returns. # # By setting up timeout to listenfd and wrap accept in a loop is # a solution but not a perfect one. connfd, remote_addr = listenfd.accept() if not os.fork(): # close listen fd in child process # not actual close, just minus its reference count # by one listenfd.close() tools.str_echo(connfd) connfd.close() sys.exit(0) connfd.close()
def __init__(self): self.processes = {} # Stop all processes as last part of mainloop termination. main.signals['shutdown-after'].connect(self.stopall) main.signals['unix-signal'].connect(self._sigchld_handler) # Set SA_RESTART bit for the signal, which restarts any interrupted # system calls -- however, select (at least on Linux) is NOT restarted # for reasons described at: # http://lkml.indiana.edu/hypermail/linux/kernel/0003.2/0336.html # # We do this early (which is effectively at import time, because # _Supervisor() gets instantiated at import) so that child processes # can be created before the main loop is started, and their # termination won't interrupt any system calls. if sys.hexversion >= 0x02060000: # Python 2.6+ has signal.siginterrupt() signal.siginterrupt(signal.SIGCHLD, False) elif sys.hexversion >= 0x02050000: # Python 2.5 import ctypes, ctypes.util libc = ctypes.util.find_library('c') ctypes.CDLL(libc).siginterrupt(signal.SIGCHLD, 0) else: # Python 2.4- is not supported. raise SystemError('kaa.base requires Python 2.5 or later')
def __register_sighandler(self): # Register a SIGUSR1 handler. def handler(signum, frame): Ping.set(True) signal.signal(signal.SIGUSR1, handler) # Don't interrupt system cals on SIGUSR1. signal.siginterrupt(signal.SIGUSR1, False)
def open(self): """ Fork a child nethack process into a pty and setup its stdin and stdout """ (self.pid, self.pipe) = os.forkpty() if self.pid == 0: # I'm the child process in a fake pty. I need to replace myself # with an instance of nethack. # # NOTE: The '--proxy' argument doesn't seem to do anything though # it's used by dgamelaunch which is a bit confusing. However, # without *some* argument execvp doesn't seem to like nethack and # launches a shell instead? It's quite confusing. if self.debug: os.execvpe("nethack", ["--proxy", "-D"], os.environ) else: os.execvpe("nethack", ["--proxy"], os.environ) else: # Before we do anything else, it's time to establish some boundries signal.siginterrupt(signal.SIGCHLD, True) signal.signal(signal.SIGCHLD, self._close) # When my tty resizes, the child's pty should resize too. signal.signal(signal.SIGWINCH, self.resize_child) # Setup out input/output proxies self.stdout = os.fdopen(self.pipe, "rb", 0) self.stdin = os.fdopen(self.pipe, "wb", 0) # Set the initial size of the child pty to my own size. self.resize_child()
def timed_wait(p, timeout): signal(SIGALRM, raise_timeout_exception) siginterrupt(SIGALRM, False) alarm(timeout) rc = p.wait() alarm(0) return rc
def stop(self, signum=None, _unused=None): """Stop the consumer from consuming by calling BasicCancel and setting our state. :param int signum: The signal received :param frame _unused: The stack frame from when the signal was called """ LOGGER.debug('Stop called in state: %s', self.state_description) if self.is_stopped: LOGGER.warning('Stop requested but consumer is already stopped') return elif self.is_shutting_down: LOGGER.warning('Stop requested, consumer is already shutting down') return elif self.is_waiting_to_shutdown: LOGGER.warning('Stop requested but already waiting to shut down') return # Stop consuming and close AMQP connections self.shutdown_connections() # Wait until the consumer has finished processing to shutdown if self.is_processing: LOGGER.info('Waiting for consumer to finish processing') self.set_state(self.STATE_STOP_REQUESTED) if signum == signal.SIGTERM: signal.siginterrupt(signal.SIGTERM, False) return
def add_signal_handler(self, sig, callback, *args): """Add a handler for a signal. UNIX only. Raise ValueError if the signal number is invalid or uncatchable. Raise RuntimeError if there is a problem setting up the handler. """ if (coroutines.iscoroutine(callback) or coroutines.iscoroutinefunction(callback)): raise TypeError("coroutines cannot be used " "with add_signal_handler()") self._check_signal(sig) self._check_closed() try: # set_wakeup_fd() raises ValueError if this is not the # main thread. By calling it early we ensure that an # event loop running in another thread cannot add a signal # handler. signal.set_wakeup_fd(self._csock.fileno()) except (ValueError, OSError) as exc: raise RuntimeError(str(exc)) handle = events.Handle(callback, args, self) self._signal_handlers[sig] = handle try: if compat.PY33: # On Python 3.3 and newer, the C signal handler writes the # signal number into the wakeup file descriptor and then calls # Py_AddPendingCall() to schedule the Python signal handler. # # Register a dummy signal handler to ask Python to write the # signal number into the wakup file descriptor. # _process_self_data() will read signal numbers from this file # descriptor to handle signals. signal.signal(sig, _sighandler_noop) else: # On Python 3.2 and older, the C signal handler first calls # Py_AddPendingCall() to schedule the Python signal handler, # and then write a null byte into the wakeup file descriptor. signal.signal(sig, self._handle_signal) # Set SA_RESTART to limit EINTR occurrences. signal.siginterrupt(sig, False) except (RuntimeError, OSError) as exc: # On Python 2, signal.signal(signal.SIGKILL, signal.SIG_IGN) raises # RuntimeError(22, 'Invalid argument'). On Python 3, # OSError(22, 'Invalid argument') is raised instead. exc_type, exc_value, tb = sys.exc_info() del self._signal_handlers[sig] if not self._signal_handlers: try: signal.set_wakeup_fd(-1) except (ValueError, OSError) as nexc: logger.info('set_wakeup_fd(-1) failed: %s', nexc) if isinstance(exc, RuntimeError) or exc.errno == errno.EINVAL: raise RuntimeError('sig {0} cannot be caught'.format(sig)) else: reraise(exc_type, exc_value, tb)
def sigusr1_handler(self, signum, stack_frame): # Apparently the setting siginterrupt can get reset on some platforms. signal.siginterrupt(signal.SIGUSR1, False) print('Received SIGUSR1. Current stack trace:', file=sys.stderr) traceback.print_stack(stack_frame) if self.runner is not None: self.runner.debug_status()
def init_signals(self): # reset signal handlers to defaults [signal.signal(s, signal.SIG_DFL) for s in self.SIGNALS] # einhorn will send SIGUSR2 to request a graceful shutdown signal.signal(signal.SIGUSR2, self.start_graceful_shutdown) signal.siginterrupt(signal.SIGUSR2, False)
def _child_handler(self, sig, stack): pid, sts = os.waitpid(-1, os.WNOHANG) proc = self._procs.get(pid) if proc is not None: self._proc_status(proc, sts) signal.signal(SIGCHLD, self._child_handler) signal.siginterrupt(SIGCHLD, False)
def __init__(self, bot_id): self.__log_buffer = [] self.parameters = Parameters() self.__error_retries_counter = 0 self.__source_pipeline = None self.__destination_pipeline = None self.logger = None try: version_info = sys.version.splitlines()[0].strip() self.__log_buffer.append(('info', '{} initialized with id {} and version ' '{} as process {}.' ''.format(self.__class__.__name__, bot_id, version_info, os.getpid()))) self.__log_buffer.append(('debug', 'Library path: %r.' % __file__)) self.__load_defaults_configuration() self.__load_system_configuration() self.__check_bot_id(bot_id) self.__bot_id = bot_id if self.parameters.logging_handler == 'syslog': syslog = self.parameters.logging_syslog else: syslog = False self.logger = utils.log(self.__bot_id, syslog=syslog, log_path=self.parameters.logging_path, log_level=self.parameters.logging_level) except: self.__log_buffer.append(('critical', traceback.format_exc())) self.stop() else: for line in self.__log_buffer: getattr(self.logger, line[0])(line[1]) try: self.logger.info('Bot is starting.') self.__load_runtime_configuration() self.__load_pipeline_configuration() self.__load_harmonization_configuration() self.init() self.__sighup = False signal.signal(signal.SIGHUP, self.__handle_sighup_signal) # system calls should not be interrupted, but restarted signal.siginterrupt(signal.SIGHUP, False) except Exception as exc: if self.parameters.error_log_exception: self.logger.exception('Bot initialization failed.') else: self.logger.error(utils.error_message_from_exc(exc)) self.logger.error('Bot initialization failed.') self.stop() raise
def main(): proc_pool = {} # 记录创建的所有子进程 cntl_q = mp.Queue() # 控制信息传递队列 data_q = mp.Queue() # 具体数据传递队列 exit_flag = mp.Event() # 退出标记,初始值为False # 收到SIGINT,通知proxy停止读取数据 signal(SIGINT, lambda x, y: exit_flag.set()) siginterrupt(SIGINT, False) # 启动proxy进程,后续按需启动woker进程 print 'main {} started'.format(os.getpid()) proc = mp.Process(target=proc_proxy, args=(cntl_q, data_q, exit_flag)) proc.start() proc_pool[proc.pid] = proc print 'proxy {} started'.format(proc.pid) while True: item = cntl_q.get() if item['event'] == 'data': proc = mp.Process(target=proc_worker, args=(cntl_q, data_q)) proc.start() proc_pool[proc.pid] = proc print 'worker {} started'.format(proc.pid) elif item['event'] == 'exit': proc = proc_pool.pop(item['pid']) proc.join() print 'child {} stopped'.format(item['pid']) else: print 'It\'s impossible !' if not proc_pool: # 所有子进程均已退出 break print 'main {} stopped'.format(os.getpid())
def main(): """Parses the arguments and call worker()""" # Set the signal handler for s in [signal.SIGINT, signal.SIGTERM]: signal.signal(s, shutdown) signal.siginterrupt(s, False) parser, _ = utils.create_argparser(__doc__) parser.add_argument( '--sensor', metavar='SENSOR[:SENSOR]', help='sensor to check, optionally with a long name, defaults to all.', ) parser.add_argument( '--directory', metavar='DIR', help='base directory (defaults to /ivre/passiverecon/).', default="/ivre/passiverecon/", ) parser.add_argument( '--progname', metavar='PROG', help='Program to run (defaults to ivre passiverecon2db).', default="ivre passiverecon2db", ) args = parser.parse_args() if args.sensor is not None: SENSORS.update(dict([args.sensor.split(':', 1) if ':' in args.sensor else [args.sensor, args.sensor]])) sensor = args.sensor.split(':', 1)[0] else: sensor = None worker(args.progname, args.directory, sensor=sensor)
def __init_signal(self): try: signal.signal(signal.SIGCHLD, signal.SIG_DFL) signal.siginterrupt(signal.SIGCHLD, False) signal.signal(signal.SIGTERM, MasterD.sig_term_handler) except: raise
def hup_hook(signal_or_callable=signal.SIGTERM, verbose=False): """ Register a signal handler for `signal.SIGHUP` that checks for modified files and only acts if at least one modified file is found. @type signal_or_callable: str, int or callable @param signal_or_callable: You can pass either a signal or a callable. The signal can be specified by name or number. If specifying by name, the 'SIG' portion is optional. For example, valid values for SIGINT include 'INT', 'SIGINT' and `signal.SIGINT`. Alternatively, you can pass a callable that will be called with the list of changed files. So the call signature should be `func(list)`. The return value of the callable is ignored. @type verbose: bool or callable @param verbose: Defaults to False. True indicates that a message should be printed. You can also pass a callable such as log.info. """ #noinspection PyUnusedLocal def handle_hup(signum, frame): changed = modified() if changed: if callable(signal_or_callable): func = signal_or_callable args = (changed,) op = 'Calling' try: name = signal_or_callable.__name__ except Exception: name = str(signal_or_callable) else: if isinstance(signal_or_callable, int): name = str(signal_or_callable) signum = signal_or_callable if verbose: for item in dir(signal): if item.startswith('SIG') and getattr(signal, item) == signal_or_callable: name = item break else: name = signal_or_callable if signal_or_callable.startswith('SIG') else 'SIG' + signal_or_callable signum = getattr(signal, name) func = os.kill args = (os.getpid(), signum) op = 'Sending' if verbose: more = ' and {0} other files'.format(len(changed)) if len(changed) > 1 else '' message = '{0} {1} because {2}{3} changed'.format(op, name, changed[0], more) if callable(verbose): #noinspection PyCallingNonCallable verbose(message) else: print(message) func(*args) files() signal.signal(signal.SIGHUP, handle_hup) signal.siginterrupt(signal.SIGHUP, False)
def __init__(self, interval=0.01, mode='virtual'): self.interval = interval self.mode = mode assert mode in Collector.MODES timer, sig = Collector.MODES[self.mode] signal.signal(sig, self.handler) signal.siginterrupt(sig, False) self.reset()
def set_sigchld_handler(self): # TODO: find out whether set_wakeup_fd still works if the default # signal handler is used (I'm pretty sure it doesn't work if the # signal is ignored). signal.signal(signal.SIGCHLD, self.handle_sigchld) # This should keep reads and writes from getting EINTR. if hasattr(signal, 'siginterrupt'): signal.siginterrupt(signal.SIGCHLD, False)
def setupHandler(self): self.fd = socket.socketpair(socket.AF_UNIX,socket.SOCK_STREAM,0) if not self.fd: return -1 Signal.fds[self.signum] = self signal.signal(self.signum,self.handler) signal.siginterrupt(self.signum,False) return 0
def mount(self, args): '''Mount a backup repository as a FUSE filesystem. This subcommand allows you to access backups in an Obnam backup repository as normal files and directories. Each backed up file or directory can be viewed directly, using a graphical file manager or command line tools. Example: To mount your backup repository: mkdir my-fuse obnam mount --viewmode multiple --to my-fuse You can then access the backup using commands such as these: ls -l my-fuse ls -l my-fuse/latest diff -u my-fuse/latest/home/liw/README ~/README You can also restore files by copying them from the my-fuse directory: cp -a my-fuse/12765/Maildir ~/Maildir.restored To un-mount: fusermount -u my-fuse ''' if not hasattr(fuse, 'fuse_python_api'): raise obnamlib.Error('Failed to load module "fuse", ' 'try installing python-fuse') self.app.settings.require('repository') self.app.settings.require('client-name') self.app.settings.require('to') self.repo = self.app.open_repository() self.repo.open_client(self.app.settings['client-name']) self.mountroot = (['/'] + self.app.settings['root'] + args)[-1] if self.mountroot != '/': self.mountroot = self.mountroot.rstrip('/') logging.debug('FUSE Mounting %s@%s:%s to %s', self.app.settings['client-name'], self.app.settings['generation'], self.mountroot, self.app.settings['to']) try: ObnamFuseOptParse.obnam = self fs = ObnamFuse(obnam=self, parser_class=ObnamFuseOptParse) signal.signal(signal.SIGUSR1, lambda s,f: fs.sigUSR1()) signal.siginterrupt(signal.SIGUSR1, False) fs.flags = 0 fs.multithreaded = 0 fs.parse() fs.main() except fuse.FuseError, e: raise obnamlib.Error(repr(e))
def setup_signals(self): """ Register signal handlers """ signal.signal(signal.SIGTERM, self.sigterm_handler) signal.signal(signal.SIGINT, self.sigterm_handler) signal.signal(signal.SIGHUP, self.sighup_handler) signal.siginterrupt(signal.SIGHUP, False) logger.info("Set up signal handlers")
def run(self): signal(SIGTERM, self._sigterm) signal(SIGINT, self._sigterm) # Try to avoid stacktraces from interrupted signal calls siginterrupt(SIGTERM, False) siginterrupt(SIGINT, False) if self.options.restart: if os.path.exists(self.config_filename): os.remove(self.config_filename) self._loadConfig() if self.options.batchsize <= 0: self.parser.error('Invalid argument for --batchsize parameter - must be positive') if self.options.sleep < 0: self.parser.error('Invalid argument for --sleep parameter') if not self.options.fetchArgs: if not self.options.evtuser or self.options.evtpass is None: self.parser.error('Required arguments --evtuser and --evtpass must be provided when using ' '--dont-fetch-args') else: zem = self.dmd.ZenEventManager self.options.evthost = zem.host self.options.evtport = zem.port self.options.evtuser = zem.username self.options.evtpass = zem.password self.options.evtdb = zem.database conn = None publisher = None try: conn = connect(host=self.options.evthost, user=self.options.evtuser, passwd=self.options.evtpass, db=self.options.evtdb, port=self.options.evtport, cursorclass=DictCursor, use_unicode=True) conn.autocommit(1) publisher = getUtility(IQueuePublisher) # Migrate status self._migrate_events(conn, publisher, True) # Migrate history self._migrate_events(conn, publisher, False) except Exception as e: if log.isEnabledFor(logging.DEBUG): log.exception('Error migrating events') print >>sys.stderr, "Failed to migrate events: %s" % e finally: if publisher: publisher.close() if conn: conn.close()
def main(argv, config): g.running = 'server' parser = ArgumentParser(prog=argv.pop(0)) parser.add_argument('--debug', action='store_true') options = parser.parse_args(argv) # all forks belong to the same happy family try: os.setpgrp() except OSError: print( "Failed to create process group - there is probably already one (daemontools).", file=sys.stderr) # Set a low (but not too low) open file limit to make # dispatch.update_valid_fds faster. # The runners will set the highest limit they can # before actually running any methods. r1, r2 = resource.getrlimit(resource.RLIMIT_NOFILE) r1 = min(r1, r2, 1024) resource.setrlimit(resource.RLIMIT_NOFILE, (r1, r2)) # Start the board in a separate process so it can't interfere. # Even if it dies we don't care. try: if not isinstance(config.board_listen, tuple): # Don't bother if something is already listening. check_socket(config.board_listen) Process(target=board.run, args=(config, ), name='board').start() except Exception: pass iowrapper.main() # setup statmsg sink and tell address using ENV statmsg_rd, statmsg_wr = socket.socketpair(socket.AF_UNIX, socket.SOCK_DGRAM) os.environ['BD_STATUS_FD'] = str(statmsg_wr.fileno()) def buf_up(fh, opt): sock = socket.fromfd(fh.fileno(), socket.AF_UNIX, socket.SOCK_DGRAM) sock.setsockopt(socket.SOL_SOCKET, opt, 256 * 1024) # does not close fh, because fromfd dups the fd (but not the underlying socket) sock.close() buf_up(statmsg_wr, socket.SO_SNDBUF) buf_up(statmsg_rd, socket.SO_RCVBUF) t = DeadlyThread(target=statmsg_sink, args=(statmsg_rd, ), name="statmsg sink") t.daemon = True t.start() # do all main-stuff, i.e. run server sys.stdout = autoflush.AutoFlush(sys.stdout) sys.stderr = autoflush.AutoFlush(sys.stderr) atexit.register(exitfunction) signal.signal(signal.SIGTERM, exitfunction) signal.signal(signal.SIGINT, exitfunction) signal.signal(signal.SIGUSR1, siginfo) signal.siginterrupt(signal.SIGUSR1, False) if hasattr(signal, 'SIGINFO'): signal.signal(signal.SIGINFO, siginfo) signal.siginterrupt(signal.SIGINFO, False) if isinstance(config.listen, tuple): server = ThreadedHTTPServer(config.listen, XtdHandler) else: check_socket(config.listen) # We want the socket to be world writeable, protect it with dir permissions. u = os.umask(0) server = ThreadedUnixHTTPServer(config.listen, XtdHandler) os.umask(u) if config.get('urd_local'): from accelerator import urd t = DeadlyThread(target=urd.main, args=(['urd', '--quiet', '--allow-passwordless'], config), name='urd') t.daemon = True t.start() ctrl = control.Main(config, options, config.url) print() ctrl.print_workdirs() print() XtdHandler.ctrl = ctrl job_tracking[None].workdir = ctrl.target_workdir for n in ( "project_directory", "result_directory", "input_directory", ): v = config.get(n) n = n.replace("_", " ") print("%17s: %s" % ( n, v, )) for n in ( "board", "urd", ): v = config.get(n + '_listen') if v and not config.get(n + '_local', True): extra = ' (remote)' else: extra = '' print("%17s: %s%s" % ( n, v, extra, )) print() print("Serving on %s\n" % (config.listen, ), file=sys.stderr) server.serve_forever()
def init_signals(self): [signal.signal(s, signal.SIG_DFL) for s in self.SIGNALS] signal.signal(signal.SIGQUIT, self.handle_quit) signal.signal(signal.SIGTERM, self.handle_exit) signal.signal(signal.SIGINT, self.handle_quit) signal.siginterrupt(signal.SIGTERM, False)
def printTempStats(): init_strings = "\n\n" + str(datetime.now()) + " TEMP" printStats(init_strings) def sigquitHandler(signum, frame): init_strings = "\n\n" + str(datetime.now()) + " END" printStats(init_strings) stdout.write("\n") sys.exit(0) signal.signal(signal.SIGINT, sigquitHandler) signal.siginterrupt(signal.SIGINT, True) #init some variables command_limit_one= \ "http://ws.audioscrobbler.com/2.0/?method=user.getfriends"\ "&user={0}&limit=1&page={1}&api_key={2}" command_user_info = \ "http://ws.audioscrobbler.com/2.0/?method=user.getinfo"\ "&user={0}&api_key={1}" command_user_shouts = \ "http://ws.audioscrobbler.com/2.0/?method=user.getshouts"\ "&user={0}&limit={1}&api_key={2}" api_key = 'cda9140cf81af12206d411e1d420af18' #team_amz's API Key
def main(args, net, datadir_path, merged_urls, worker_endpoint): try: print 'p2pool (version %s)' % (p2pool.__version__, ) print @defer.inlineCallbacks def connect_p2p(): # connect to bitcoind over bitcoin-p2p print '''Testing bitcoind P2P connection to '%s:%s'...''' % ( args.bitcoind_address, args.bitcoind_p2p_port) factory = bitcoin_p2p.ClientFactory(net.PARENT) reactor.connectTCP(args.bitcoind_address, args.bitcoind_p2p_port, factory) yield factory.getProtocol() # waits until handshake is successful print ' ...success!' print defer.returnValue(factory) if args.testnet: # establish p2p connection first if testnet so bitcoind can work without connections factory = yield connect_p2p() # connect to bitcoind over JSON-RPC and do initial getmemorypool url = '%s://%s:%i/' % ('https' if args.bitcoind_rpc_ssl else 'http', args.bitcoind_address, args.bitcoind_rpc_port) print '''Testing bitcoind RPC connection to '%s' with username '%s'...''' % ( url, args.bitcoind_rpc_username) bitcoind = jsonrpc.Proxy( url, dict(Authorization='Basic ' + base64.b64encode(args.bitcoind_rpc_username + ':' + args.bitcoind_rpc_password)), timeout=30) yield helper.check(bitcoind, net) temp_work = yield helper.getwork(bitcoind) bitcoind_warning_var = variable.Variable(None) @defer.inlineCallbacks def poll_warnings(): errors = (yield deferral.retry('Error while calling getmininginfo:')( bitcoind.rpc_getmininginfo)())['errors'] bitcoind_warning_var.set(errors if errors != '' else None) yield poll_warnings() task.LoopingCall(poll_warnings).start(20 * 60) print ' ...success!' print ' Current block hash: %x' % (temp_work['previous_block'], ) print ' Current block height: %i' % (temp_work['height'] - 1, ) print if not args.testnet: factory = yield connect_p2p() print 'Determining payout address...' if args.pubkey_hash is None: address_path = os.path.join(datadir_path, 'cached_payout_address') if os.path.exists(address_path): with open(address_path, 'rb') as f: address = f.read().strip('\r\n') print ' Loaded cached address: %s...' % (address, ) else: address = None if address is not None: res = yield deferral.retry( 'Error validating cached address:', 5)(lambda: bitcoind.rpc_validateaddress(address))() if not res['isvalid'] or not res['ismine']: print ' Cached address is either invalid or not controlled by local bitcoind!' address = None if address is None: print ' Getting payout address from bitcoind...' address = yield deferral.retry( 'Error getting payout address from bitcoind:', 5)(lambda: bitcoind.rpc_getaccountaddress('p2pool'))() with open(address_path, 'wb') as f: f.write(address) my_pubkey_hash = bitcoin_data.address_to_pubkey_hash( address, net.PARENT) else: my_pubkey_hash = args.pubkey_hash print ' ...success! Payout address:', bitcoin_data.pubkey_hash_to_address( my_pubkey_hash, net.PARENT) print ss = p2pool_data.ShareStore(os.path.join(datadir_path, 'shares.'), net) shares = {} known_verified = set() print "Loading shares..." for i, (mode, contents) in enumerate(ss.get_shares()): if mode == 'share': contents.time_seen = 0 shares[contents.hash] = contents if len(shares) % 1000 == 0 and shares: print " %i" % (len(shares), ) elif mode == 'verified_hash': known_verified.add(contents) else: raise AssertionError() print " ...done loading %i shares (%i verified)!" % ( len(shares), len(known_verified)) print print 'Initializing work...' node = p2pool_node.Node(factory, bitcoind, shares.values(), known_verified, net) yield node.start() for share_hash in shares: if share_hash not in node.tracker.items: ss.forget_share(share_hash) for share_hash in known_verified: if share_hash not in node.tracker.verified.items: ss.forget_verified_share(share_hash) del shares, known_verified node.tracker.removed.watch(lambda share: ss.forget_share(share.hash)) node.tracker.verified.removed.watch( lambda share: ss.forget_verified_share(share.hash)) def save_shares(): for share in node.tracker.get_chain( node.best_share_var.value, min(node.tracker.get_height(node.best_share_var.value), 2 * net.CHAIN_LENGTH)): ss.add_share(share) if share.hash in node.tracker.verified.items: ss.add_verified_hash(share.hash) task.LoopingCall(save_shares).start(60) print ' ...success!' print print 'Joining p2pool network using port %i...' % (args.p2pool_port, ) @defer.inlineCallbacks def parse(x): if ':' in x: ip, port = x.split(':') defer.returnValue(((yield reactor.resolve(ip)), int(port))) else: defer.returnValue(((yield reactor.resolve(x)), net.P2P_PORT)) addrs = {} if os.path.exists(os.path.join(datadir_path, 'addrs')): try: with open(os.path.join(datadir_path, 'addrs'), 'rb') as f: addrs.update( dict((tuple(k), v) for k, v in json.loads(f.read()))) except: print >> sys.stderr, 'error parsing addrs' for addr_df in map(parse, net.BOOTSTRAP_ADDRS): try: addr = yield addr_df if addr not in addrs: addrs[addr] = (0, time.time(), time.time()) except: log.err() connect_addrs = set() for addr_df in map(parse, args.p2pool_nodes): try: connect_addrs.add((yield addr_df)) except: log.err() node.p2p_node = p2pool_node.P2PNode( node, port=args.p2pool_port, max_incoming_conns=args.p2pool_conns, addr_store=addrs, connect_addrs=connect_addrs, desired_outgoing_conns=args.p2pool_outgoing_conns, ) node.p2p_node.start() def save_addrs(): with open(os.path.join(datadir_path, 'addrs'), 'wb') as f: f.write(json.dumps(node.p2p_node.addr_store.items())) task.LoopingCall(save_addrs).start(60) print ' ...success!' print if args.upnp: @defer.inlineCallbacks def upnp_thread(): while True: try: is_lan, lan_ip = yield ipdiscover.get_local_ip() if is_lan: pm = yield portmapper.get_port_mapper() yield pm._upnp.add_port_mapping( lan_ip, args.p2pool_port, args.p2pool_port, 'p2pool', 'TCP') except defer.TimeoutError: pass except: if p2pool.DEBUG: log.err(None, 'UPnP error:') yield deferral.sleep(random.expovariate(1 / 120)) upnp_thread() # start listening for workers with a JSON-RPC server print 'Listening for workers on %r port %i...' % (worker_endpoint[0], worker_endpoint[1]) wb = work.WorkerBridge(node, my_pubkey_hash, args.donation_percentage, merged_urls, args.worker_fee) web_root = web.get_web_root(wb, datadir_path, bitcoind_warning_var) worker_interface.WorkerInterface(wb).attach_to( web_root, get_handler=lambda request: request.redirect('/static/')) deferral.retry('Error binding to worker port:', traceback=False)( reactor.listenTCP)(worker_endpoint[1], server.Site(web_root), interface=worker_endpoint[0]) with open(os.path.join(os.path.join(datadir_path, 'ready_flag')), 'wb') as f: pass print ' ...success!' print # done! print 'Started successfully!' print 'Go to http://127.0.0.1:%i/ to view graphs and statistics!' % ( worker_endpoint[1], ) if args.donation_percentage > 0.51: print '''Donating %.1f%% of work towards P2Pool's development. Thanks for the tip!''' % ( args.donation_percentage, ) elif args.donation_percentage < 0.49: print '''Donating %.1f%% of work towards P2Pool's development. Please donate to encourage further development of P2Pool!''' % ( args.donation_percentage, ) else: print '''Donating %.1f%% of work towards P2Pool's development. Thank you!''' % ( args.donation_percentage, ) print 'You can increase this amount with --give-author argument! (or decrease it, if you must)' print if hasattr(signal, 'SIGALRM'): signal.signal( signal.SIGALRM, lambda signum, frame: reactor.callFromThread( sys.stderr.write, 'Watchdog timer went off at:\n' + ''. join(traceback.format_stack()))) signal.siginterrupt(signal.SIGALRM, False) task.LoopingCall(signal.alarm, 30).start(1) if args.irc_announce: from twisted.words.protocols import irc class IRCClient(irc.IRCClient): nickname = 'p2pool%02i' % (random.randrange(100), ) channel = net.ANNOUNCE_CHANNEL def lineReceived(self, line): if p2pool.DEBUG: print repr(line) irc.IRCClient.lineReceived(self, line) def signedOn(self): self.in_channel = False irc.IRCClient.signedOn(self) self.factory.resetDelay() self.join(self.channel) @defer.inlineCallbacks def new_share(share): if not self.in_channel: return if share.pow_hash <= share.header[ 'bits'].target and abs(share.timestamp - time.time()) < 10 * 60: yield deferral.sleep(random.expovariate(1 / 60)) message = '\x02%s BLOCK FOUND by %s! %s%064x' % ( net.NAME.upper(), bitcoin_data.script2_to_address( share.new_script, net.PARENT), net.PARENT.BLOCK_EXPLORER_URL_PREFIX, share.header_hash) if all('%x' % (share.header_hash, ) not in old_message for old_message in self.recent_messages): self.say(self.channel, message) self._remember_message(message) self.watch_id = node.tracker.verified.added.watch( new_share) self.recent_messages = [] def joined(self, channel): self.in_channel = True def left(self, channel): self.in_channel = False def _remember_message(self, message): self.recent_messages.append(message) while len(self.recent_messages) > 100: self.recent_messages.pop(0) def privmsg(self, user, channel, message): if channel == self.channel: self._remember_message(message) def connectionLost(self, reason): node.tracker.verified.added.unwatch(self.watch_id) print 'IRC connection lost:', reason.getErrorMessage() class IRCClientFactory(protocol.ReconnectingClientFactory): protocol = IRCClient reactor.connectTCP("irc.freenode.net", 6667, IRCClientFactory()) @defer.inlineCallbacks def status_thread(): last_str = None last_time = 0 while True: yield deferral.sleep(3) try: height = node.tracker.get_height(node.best_share_var.value) this_str = 'P2Pool: %i shares in chain (%i verified/%i total) Peers: %i (%i incoming)' % ( height, len(node.tracker.verified.items), len(node.tracker.items), len(node.p2p_node.peers), sum(1 for peer in node.p2p_node.peers.itervalues() if peer.incoming), ) + (' FDs: %i R/%i W' % (len(reactor.getReaders()), len(reactor.getWriters())) if p2pool.DEBUG else '') datums, dt = wb.local_rate_monitor.get_datums_in_last() my_att_s = sum(datum['work'] / dt for datum in datums) this_str += '\n Local: %sH/s in last %s Local dead on arrival: %s Expected time to share: %s' % ( math.format(int(my_att_s)), math.format_dt(dt), math.format_binomial_conf( sum(1 for datum in datums if datum['dead']), len(datums), 0.95), math.format_dt(2**256 / node.tracker.items[ node.best_share_var.value].max_target / my_att_s) if my_att_s and node.best_share_var.value else '???', ) if height > 2: (stale_orphan_shares, stale_doa_shares), shares, _ = wb.get_stale_counts() stale_prop = p2pool_data.get_average_stale_prop( node.tracker, node.best_share_var.value, min(60 * 60 // net.SHARE_PERIOD, height)) real_att_s = p2pool_data.get_pool_attempts_per_second( node.tracker, node.best_share_var.value, min(height - 1, 60 * 60 // net.SHARE_PERIOD)) / (1 - stale_prop) this_str += '\n Shares: %i (%i orphan, %i dead) Stale rate: %s Efficiency: %s Current payout: %.4f %s' % ( shares, stale_orphan_shares, stale_doa_shares, math.format_binomial_conf( stale_orphan_shares + stale_doa_shares, shares, 0.95), math.format_binomial_conf( stale_orphan_shares + stale_doa_shares, shares, 0.95, lambda x: (1 - x) / (1 - stale_prop)), node.get_current_txouts().get( bitcoin_data.pubkey_hash_to_script2( my_pubkey_hash), 0) * 1e-8, net.PARENT.SYMBOL, ) this_str += '\n Pool: %sH/s Stale rate: %.1f%% Expected time to block: %s' % ( math.format(int(real_att_s)), 100 * stale_prop, math.format_dt( 2**256 / node.bitcoind_work.value['bits'].target / real_att_s), ) for warning in p2pool_data.get_warnings( node.tracker, node.best_share_var.value, net, bitcoind_warning_var.value, node.bitcoind_work.value): print >> sys.stderr, '#' * 40 print >> sys.stderr, '>>> Warning: ' + warning print >> sys.stderr, '#' * 40 if this_str != last_str or time.time() > last_time + 15: print this_str last_str = this_str last_time = time.time() except: log.err() status_thread() except: reactor.stop() log.err(None, 'Fatal error:')
def main(): """Main routine. Do the work.""" # # First, write a PID so we can monitor the process # pid = str(os.getpid()) # pidfile = "/var/run/bspdserver.pid" # # if os.path.isfile(pidfile): # print "%s already exists, exiting" % pidfile # sys.exit() # else: # file(pidfile, 'w').write(pid) # Some logging preamble logging.debug('\n\n-=- Starting new BSDP server session -=-\n') # We are changing nbiimages for use by other functions global nbiimages # Instantiate a basic pydhcplib DhcpServer class using netopts (listen port, # reply port and listening IP) server = Server(netopt) # Do a one-time discovery of all available NBIs on the server. NBIs added # after the server was started will not be picked up until after a restart nbiimages, nbisources = getNbiOptions(tftprootpath) def scan_nbis(signal, frame): global nbiimages logging.debug('[========= Updating boot images list =========]') nbiimages, nbisources = getNbiOptions(tftprootpath) for nbi in nbisources: logging.debug(nbi) logging.debug('[========= End updated list =========]') signal.signal(signal.SIGUSR1, scan_nbis) signal.siginterrupt(signal.SIGUSR1, False) # Print the full list of eligible NBIs to the log logging.debug('[========= Using the following boot images =========]') for nbi in nbisources: logging.debug(nbi) logging.debug('[========= End boot image listing =========]') # Loop while the looping's good. while True: # Listen for DHCP packets. Since select() is used upstream we need to # catch the EINTR signal it trips on when we receive a USR1 signal to # reload the nbiimages list. try: packet = server.GetNextDhcpPacket() except select.error, e: if e[0] != errno.EINTR: raise try: # Check to see if any vendor_encapsulated_options are present if len(packet.GetOption('vendor_encapsulated_options')) > 1: # If we have vendor_encapsulated_options check for a value of 1 # which in BSDP terms means the packet is a BSDP[LIST] request if packet.GetOption('vendor_encapsulated_options')[2] == 1: logging.debug( '-=========================================-') logging.debug('Got BSDP INFORM[LIST] packet: ') # Pass ack() the matching packet, defaultnbi and 'list' bsdplistack, clientip, replyport = ack( packet, defaultnbi, 'list') # Once we have a finished DHCP packet, send it to the client server.SendDhcpPacketTo(bsdplistack, str(clientip), replyport) # If the vendor_encapsulated_options BSDP type is 2, we process # the packet as a BSDP[SELECT] request elif packet.GetOption('vendor_encapsulated_options')[2] == 2: logging.debug( '-=========================================-') logging.debug('Got BSDP INFORM[SELECT] packet: ') bsdpselectack, selectackclientip, selectackreplyport = \ ack(packet, None, 'select') # Once we have a finished DHCP packet, send it to the client server.SendDhcpPacketTo(bsdpselectack, str(selectackclientip), selectackreplyport) # If the packet length is 7 or less, move on, BSDP packets are # at least 8 bytes long. elif len(packet.GetOption('vendor_encapsulated_options')) <= 7: pass except: # Dump tracbacks to stderr and carry on traceback.print_exc() pass
def _swap_handler(self, signum, signal_handler): self._old_handlers[signum] = signal.getsignal(signum) signal.signal(signum, signal_handler) if not platform._is_win: signal.siginterrupt(signum, False)
def __init__(self, bot_id): self.__log_buffer = [] self.parameters = Parameters() self.__error_retries_counter = 0 self.__source_pipeline = None self.__destination_pipeline = None self.logger = None try: version_info = sys.version.splitlines()[0].strip() self.__log_buffer.append( ('info', '{} initialized with id {} and version ' '{} as process {}.' ''.format(self.__class__.__name__, bot_id, version_info, os.getpid()))) self.__log_buffer.append(('debug', 'Library path: %r.' % __file__)) self.__load_defaults_configuration() self.__load_system_configuration() self.__check_bot_id(bot_id) self.__bot_id = bot_id if self.parameters.logging_handler == 'syslog': syslog = self.parameters.logging_syslog else: syslog = False self.logger = utils.log(self.__bot_id, syslog=syslog, log_path=self.parameters.logging_path, log_level=self.parameters.logging_level) except: self.__log_buffer.append(('critical', traceback.format_exc())) self.stop() else: for line in self.__log_buffer: getattr(self.logger, line[0])(line[1]) try: self.logger.info('Bot is starting.') self.__load_runtime_configuration() self.__load_pipeline_configuration() self.__load_harmonization_configuration() if not getattr(self.parameters, 'enabled', True): self.logger.warn('The bot was disabled by configuration. ' 'It will not be started as long as this ' 'configuration is present.') self.stop() self.init() self.__sighup = False signal.signal(signal.SIGHUP, self.__handle_sighup_signal) # system calls should not be interrupted, but restarted signal.siginterrupt(signal.SIGHUP, False) except Exception as exc: if self.parameters.error_log_exception: self.logger.exception('Bot initialization failed.') else: self.logger.error(utils.error_message_from_exc(exc)) self.logger.error('Bot initialization failed.') self.stop() raise
def run(self): signal(SIGTERM, self._sigterm) signal(SIGINT, self._sigterm) # Try to avoid stacktraces from interrupted signal calls siginterrupt(SIGTERM, False) siginterrupt(SIGINT, False) if self.options.restart: if os.path.exists(self.config_filename): os.remove(self.config_filename) if self.options.detail_file: self.populate_detail_mappings() self._loadConfig() if self.options.batchsize <= 0: self.parser.error('Invalid argument for --batchsize parameter - must be positive') if self.options.sleep < 0: self.parser.error('Invalid argument for --sleep parameter') if self.options.startoffset < 0: self.parser.error('Invalid argument for --startoffset parameter - must be non-negative') if self.options.numevents < 0: self.parser.error('Invalid argument for --numevents parameter - must be non-negative') self.firstTimeStart = None if self.options.firstTimeStart: if not self.options.firstTimeEnd: self.parser.error('Invalid argument for --firsttime-end parameter - must be specified') if self.options.firstTimeEnd: if not self.options.firstTimeStart: self.parser.error('Invalid argument for --firsttime-start parameter - must be specified') firstTimeStart = dateutil.parser.parse(self.options.firstTimeStart) firstTimeEnd = dateutil.parser.parse(self.options.firstTimeEnd) if firstTimeStart > firstTimeEnd: self.parser.error('Invalid argument for firsttime parameter - start must be earlier than end') if self.options.startoffset >= 0: self._output('disabling --startoffset...') self.options.startoffset = 0 if self.options.numevents >= 0: self._output('disabling --numevents...') self.options.numevents = 0 self.firstTimeStart = firstTimeStart self.firstTimeEnd = firstTimeEnd self._loadConfig(self.firstTimeStart.strftime('%Y%m%d-%H%M%S')) validTables = set(('status', 'history')) tablesToMigrate = re.split('\W+', self.options.tables) for table in tablesToMigrate: if table not in validTables: self.parser.error('Invalid argument for --tables parameter - must be either status or history') if not self.options.fetchArgs: if not self.options.evtuser or self.options.evtpass is None: self.parser.error('Required arguments --evtuser and --evtpass must be provided when using ' '--dont-fetch-args') else: zem = self.dmd.ZenEventManager self.options.evthost = zem.host self.options.evtport = zem.port self.options.evtuser = zem.username self.options.evtpass = zem.password self.options.evtdb = zem.database conn = None publisher = None try: conn = connect(host=self.options.evthost, user=self.options.evtuser, passwd=self.options.evtpass, db=self.options.evtdb, port=self.options.evtport, cursorclass=DictCursor, use_unicode=True) conn.autocommit(1) publisher = getUtility(IQueuePublisher) # Migrate status if 'status' in tablesToMigrate: self._migrate_events(conn, publisher, True) # Migrate history if 'history' in tablesToMigrate: self._migrate_events(conn, publisher, False) except Exception as e: if log.isEnabledFor(logging.DEBUG): log.exception('Error migrating events') print >>sys.stderr, "Failed to migrate events: %s" % e finally: if publisher: publisher.close() if conn: conn.close()
def init_signal(signal_num, signal_object, exception_class, handler): handler = functools.partial(handler, signal_object, exception_class) signal.signal(signal_num, handler) signal.siginterrupt(signal_num, False)
def __init__(self, workflowfile, args, jmax=100): self.args = args self.workflowfile = workflowfile self.workflowspec = load_workflow(workflowfile) self.workflowspec = filter_workflow(self.workflowspec, args.target_tasks, args.target_labels) if not self.workflowspec['stages']: if args.target_tasks: print( 'Apparently some of the chosen target tasks are not in the workflow' ) exit(0) print('Workflow is empty. Nothing to do') exit(0) workflow = build_dag_properties(self.workflowspec) if args.visualize_workflow: draw_workflow(self.workflowspec) self.possiblenexttask = workflow['nexttasks'] self.taskweights = workflow['weights'] self.topological_orderings = workflow['topological_ordering'] self.taskuniverse = [l['name'] for l in self.workflowspec['stages']] self.idtotask = [0 for l in self.taskuniverse] self.tasktoid = {} for i in range(len(self.taskuniverse)): self.tasktoid[self.taskuniverse[i]] = i self.idtotask[i] = self.taskuniverse[i] self.maxmemperid = [ self.workflowspec['stages'][tid]['resources']['mem'] for tid in range(len(self.taskuniverse)) ] self.cpuperid = [ self.workflowspec['stages'][tid]['resources']['cpu'] for tid in range(len(self.taskuniverse)) ] self.curmembooked = 0 self.curcpubooked = 0 self.curmembooked_backfill = 0 self.curcpubooked_backfill = 0 self.memlimit = float(args.mem_limit) # some configurable number self.cpulimit = float(args.cpu_limit) self.procstatus = { tid: 'ToDo' for tid in range(len(self.workflowspec['stages'])) } self.taskneeds = { t: set(self.getallrequirements(t)) for t in self.taskuniverse } self.stoponfailure = True self.max_jobs_parallel = int(jmax) self.scheduling_iteration = 0 self.process_list = [ ] # list of currently scheduled tasks with normal priority self.backfill_process_list = [ ] # list of curently scheduled tasks with low backfill priority (not sure this is needed) self.pid_to_psutilsproc = { } # cache of putilsproc for resource monitoring self.pid_to_files = { } # we can auto-detect what files are produced by which task (at least to some extent) self.pid_to_connections = { } # we can auto-detect what connections are opened by which task (at least to some extent) signal.signal(signal.SIGINT, self.SIGHandler) signal.siginterrupt(signal.SIGINT, False) self.nicevalues = [os.nice(0) for tid in range(len(self.taskuniverse))] self.internalmonitorcounter = 0 # internal use self.internalmonitorid = 0 # internal use self.tids_marked_toretry = [ ] # sometimes we might want to retry a failed task (simply because it was "unlucky") and we put them here self.retry_counter = [0 for tid in range(len(self.taskuniverse)) ] # we keep track of many times retried already self.semaphore_values = { self.workflowspec['stages'][tid].get('semaphore'): 0 for tid in range(len(self.taskuniverse)) if self.workflowspec['stages'][tid].get('semaphore') != None } # keeps current count of semaphores (defined in the json workflow). used to achieve user-defined "critical sections".
tempfile.write("collect regs->cs\n") tempfile.write("collect $current_task_pid\n") tempfile.write("end\n") tempfile.close() tempfile = open(tempfilename, "r") print "Tracepoint command:" print tempfile.read() tempfile.close() gdb.execute("source " + tempfilename, True, False) os.remove(tempfilename) gdb.execute("set disconnected-tracing on", True, False) gdb.execute("tstart") gdb.execute("kill", True, False) signal.signal(signal.SIGINT, sigint_handler) signal.siginterrupt(signal.SIGINT, False) #Connect to pipe gdb.execute("target tfile /sys/kernel/debug/gtpframe_pipe") def get_function_from_sym(sym): sym = sym.rstrip(os.linesep) sym_end = sym.find(" in section") function = "" if sym_end > 0: function = sym[0:sym_end] function_list = function.split(' + ') function_list_len = len(function_list) if function_list_len >= 1: function = function_list[0]
def init_signals(self): for sig in ('SIGINT', 'SIGTERM'): self.loop.add_signal_handler(getattr(signal, sig), self.stop) self.loop.add_signal_handler(signal.SIGHUP, self.reload) signal.siginterrupt(signal.SIGTERM, False)
def main(): MinimalPython.check() try: args = init_arg_parser() if args.print_version: print(f"Version {cyclone.VERSION}") sys.exit() config_file_reader = ControllerConfigFileReader(args.config_file) init_logging(config_file_reader.log_filename, args.enable_debug) with PIDControl(config_file_reader.pid_file) as pid_control, \ ControllerCommHandler(config_file_reader.comm_target, config_file_reader.comm_port, config_file_reader.poll_timeout) as comm_handler, \ SharedQueue() as result_queue, \ SharedQueue() as task_queue: if pid_control.lock(): logging.info("Started") logging.info(f"Controller PID: {pid_control.pid()}") logging.debug("Version: %s", cyclone.VERSION) signal.signal(signal.SIGINT, signal.SIG_IGN) signal.signal(signal.SIGHUP, signal_handler) signal.signal(signal.SIGTERM, signal_handler) signal.siginterrupt(signal.SIGHUP, True) signal.siginterrupt(signal.SIGINT, True) signal.siginterrupt(signal.SIGTERM, True) comm_handler.connect() request_retry_count = 0 request_retry_wait_duration = config_file_reader.request_retry_wait_duration max_num_request_retries = config_file_reader.max_num_request_retries lock_worker_state_table = multiprocessing.Lock() lock_result_queue = multiprocessing.Lock() cond_result_queue = multiprocessing.Condition( lock_result_queue) worker_count = config_file_reader.worker_count worker_ids = create_worker_ids(worker_count) worker_state_table = create_worker_state_table(worker_ids) worker_handle_dict = \ create_worker(worker_state_table, lock_worker_state_table, task_queue, result_queue, cond_result_queue) global RUN_CONDITION if not start_worker(worker_handle_dict, worker_state_table): logging.error("Not all worker are ready!") RUN_CONDITION = False while RUN_CONDITION: try: send_msg = None if not send_msg: with CriticalSection(cond_result_queue): if not result_queue.is_empty(): task_id = result_queue.pop_nowait() if task_id: logging.debug("Finished task: %s", task_id) send_msg = TaskFinished( comm_handler.fqdn, task_id) if not send_msg: found_ready_worker = False with CriticalSection(lock_worker_state_table): for worker_id in worker_state_table.keys(): if worker_handle_dict[worker_id].is_alive() \ and worker_state_table[worker_id].get_state == WorkerState.READY: found_ready_worker = True break if found_ready_worker: logging.debug('Requesting a task...') send_msg = TaskRequest(comm_handler.fqdn) else: worker_count = len(worker_state_table) worker_count_not_active = 0 for worker_id in worker_state_table.keys(): if not worker_handle_dict[ worker_id].is_alive(): worker_count_not_active += 1 if worker_count == worker_count_not_active: logging.error('No worker are alive!') RUN_CONDITION = False else: # Available worker are busy with CriticalSection(cond_result_queue): wait_timeout_result_queue = 1 cond_result_queue.wait( wait_timeout_result_queue) if result_queue.is_empty(): send_msg = Heartbeat( comm_handler.fqdn) if send_msg: if logging.root.isEnabledFor(logging.DEBUG): # TODO: remove redundant call of send_msg.to_string() logging.debug("Sending message to master: %s", send_msg.to_string()) comm_handler.send_string(send_msg.to_string()) # Check for response and process it. # Used redundant - TODO: make a class method. ################################################################################ in_raw_data = comm_handler.recv_string() if in_raw_data: logging.debug( "Retrieved message (raw data): %s", in_raw_data) in_msg = MessageFactory.create(in_raw_data) in_msg_type = in_msg.type() if MessageType.TASK_ASSIGN() == in_msg_type: task = in_msg.to_task() logging.debug( "Retrieved task assign for: %s", task.tid) task_queue.push(task) logging.debug( "Pushed task to task queue: %s", task.tid) elif MessageType.ACKNOWLEDGE() == in_msg_type: pass elif MessageType.WAIT_COMMAND() == in_msg_type: #TODO: Implement it on the master side! wait_duration = in_msg.duration logging.debug( "Retrieved Wait Command with duration: %fs", wait_duration) time.sleep(wait_duration) elif MessageType.EXIT_COMMAND() == in_msg_type: RUN_CONDITION = False logging.info( 'Retrieved exit message from master...' ) if request_retry_count: request_retry_count = 0 ################################################################################ else: if request_retry_count == max_num_request_retries: logging.info( 'Exiting, since maximum retry count is reached!' ) comm_handler.disconnect() RUN_CONDITION = False time.sleep(request_retry_wait_duration) # Check for response and process it. # Used redundant - TODO: make a class method. ################################################################################ in_raw_data = comm_handler.recv_string() if in_raw_data: logging.debug( "Retrieved message (raw data): %s", in_raw_data) in_msg = MessageFactory.create(in_raw_data) in_msg_type = in_msg.type() if MessageType.TASK_ASSIGN( ) == in_msg_type: task = in_msg.to_task() logging.debug( "Retrieved task assign for: %s", task.tid) task_queue.push(task) logging.debug( "Pushed task to task queue: %s", task.tid) elif MessageType.ACKNOWLEDGE( ) == in_msg_type: pass elif MessageType.WAIT_COMMAND( ) == in_msg_type: #TODO: Implement it on the master side! wait_duration = in_msg.duration logging.debug( "Retrieved Wait Command with duration: %fs", wait_duration) time.sleep(wait_duration) elif MessageType.EXIT_COMMAND( ) == in_msg_type: RUN_CONDITION = False logging.info( 'Retrieved exit message from master...' ) if request_retry_count: request_retry_count = 0 ################################################################################ else: logging.debug( 'No response retrieved - Reconnecting...' ) comm_handler.reconnect() request_retry_count += 1 except Exception as err: RUN_CONDITION = False exc_type, _, exc_tb = sys.exc_info() filename = os.path.split( exc_tb.tb_frame.f_code.co_filename)[1] logging.error( f"Caught exception (type: {exc_type}) in main loop: {err} " f"- {filename} (line: {exc_tb.tb_lineno})") if not RUN_CONDITION: try: logging.info("Shutting down all worker...") all_worker_down = False while not all_worker_down: found_active_worker = False for worker_id in worker_state_table.keys(): if worker_handle_dict[worker_id].is_alive(): os.kill(worker_handle_dict[worker_id].pid, signal.SIGUSR1) task_queue.push(PoisenPill()) logging.debug( "Waiting for worker to complete: %s", worker_handle_dict[worker_id].name) found_active_worker = True if not found_active_worker: all_worker_down = True logging.debug('All worker are down.') else: logging.debug( 'Waiting for worker to shutdown...') time.sleep(1) except Exception as err: logging.error( f"Caught exception terminating Worker: {err}") else: logging.error( f"Another instance might be already running (PID file: {config_file_reader.pid_file})!" ) sys.exit(1) except Exception as err: exc_type, _, exc_tb = sys.exc_info() filename = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1] logging.error( f"Exception in {filename} (line: {exc_tb.tb_lineno}): {err}") sys.exit(1) logging.info('Finished') sys.exit(0)
def __init__( self, name=None, # Name for log entries prefix=None, # Prefix for name (e.g., prefix/progname) level=INFO, # Logging level facility=local4, # Log facility debug=False, # Force level to DEBUG verbose=False, # Log to stderr, too. quiet=None, # Don't log anything on startup (See below) signals=True, # Enable debug on/off with SIGUSR1/SIGUSR2 propagate=False # Pass debug state on to child processes ): # # Handle the parameters # if name is None: name = os.path.basename(sys.argv[0]) assert type(name) == str if prefix is not None: assert type(prefix) == str name = prefix + "/" + name self.facility = facility if debug: level = DEBUG self.is_propagating = propagate # This prevents verbose() from choking on this being undefined. self.is_verbose = False if quiet is None: quiet = False forced_quiet = False else: forced_quiet = quiet self.is_quiet = quiet self.forced_debug = debug # # Inherit state from the environment # if STATE_VARIABLE in os.environ: try: depickled = pickle.loads(os.environ[STATE_VARIABLE]) facility = depickled['facility'] assert type(facility) == int level = depickled['last_level'] assert type(level) == int self.forced_debug = depickled['forced_debug'] assert type(self.forced_debug) == bool self.is_quiet = depickled['is_quiet'] assert type(self.is_quiet) == bool except Exception as ex: self.exception("Failed to decode %s '%s'" % (STATE_VARIABLE, os.environ[STATE_VARIABLE])) # # Set up the logger # self.logger = logging.getLogger(name) self.logger.propagate = False self.syslog_handler = None self.__syslog_handler_init() # Stderr self.stderr_handler = logging.StreamHandler(sys.stderr) self.stderr_handler.setFormatter( logging.Formatter(fmt='%(asctime)s %(message)s', datefmt='%Y-%m-%dT%H:%M:%S')) # Don't add this handler; verbose will cover it. # # Get everything set to go # self.verbose(verbose) self.level(level, save=True) self.set_debug(self.forced_debug) self.__update_env() # Grab signals and make them non-interrupting # TODO: How portable is this? if signals: signal.signal(signal.SIGUSR1, self.sigusr1) signal.signal(signal.SIGUSR2, self.sigusr2) signal.siginterrupt(signal.SIGUSR1, False) signal.siginterrupt(signal.SIGUSR2, False) if (not self.is_quiet) and (not forced_quiet): self.info("Started")
from ag.orbit.gui import Main from PyQt5.QtWidgets import QApplication from PyQt5.QtCore import Qt import sys import signal QApplication.setAttribute(Qt.AA_X11InitThreads) app = QApplication(sys.argv) win = Main() win.show() try: # override signals to allow for Ctr+C in command line try: signal.siginterrupt(signal.SIGCHLD, False) signal.signal(signal.SIGINT, signal.SIG_DFL) except AttributeError: pass # this blocks until application exit sys.exit(app.exec_()) except KeyboardInterrupt: pass