def _wait_for_port(self): wait_port, image, res = self.p('wait_for_port'), self.p('image'), 1 if wait_port: container_id = self._container.id container_info = self._client.containers.get(container_id) ip_address = container_info.attrs['NetworkSettings']['IPAddress'] unsupported_errors = [( errno.EHOSTUNREACH, '[{image}] Host {ip} cannot be reach. The container may exit abnormally. Container logs :\n{logs}' )] with contextlib.closing( socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock: while res != 0: res = sock.connect_ex((ip_address, wait_port)) logger.debug( '[%s] Waiting for port %d to respond (code:%d => %s).', image, wait_port, res, errorcode.get(res, '--')) unsupported_error = next( (e[1] for e in unsupported_errors if e[0] == res), None) if unsupported_error: raise DockerContainerError( unsupported_error.format( image=image, port=wait_port, signal=errorcode.get(res, '--'), ip=ip_address, logs=self._container.logs( stream=False).decode('utf-8'))) time.sleep(0.1 if res != 0 else 0) logger.debug('[%s] Port %d is now responding.', image, wait_port)
def health_check_celery(): """ Check health status of celery and redis broker :return: """ try: d = inspect().stats() if not d: capture_message('No running Celery workers were found.') return False, 'No running Celery workers were found.' except ConnectionError as e: capture_exception(e) return False, 'cannot connect to redis server' except IOError as e: msg = "Error connecting to the backend: " + str(e) if len(e.args) > 0 and errorcode.get(e.args[0]) == 'ECONNREFUSED': msg += ' Check that the Redis server is running.' capture_exception(e) return False, msg except ImportError as e: capture_exception(e) return False, str(e) except Exception: capture_exception() return False, 'celery not ok' return True, 'celery ok'
def health_check_celery(): """ Check health status of celery and redis broker :return: """ try: d = inspect().stats() if not d: sentry.captureMessage('No running Celery workers were found.') return False, 'No running Celery workers were found.' except ConnectionError as e: sentry.captureException() return False, 'cannot connect to redis server' except IOError as e: msg = "Error connecting to the backend: " + str(e) if len(e.args) > 0 and errorcode.get(e.args[0]) == 'ECONNREFUSED': msg += ' Check that the Redis server is running.' sentry.captureException() return False, msg except ImportError as e: sentry.catureException() return False, str(e) except Exception: sentry.captureException() return False, 'celery not ok' return True, 'celery ok'
def __init__(self, kernel, endpoint, sock=None, dgram=False, on_connected_callback=None, on_close_callback=None): self.kernel = kernel self.endpoint = endpoint self.dgram = dgram self._on_connected_cb = on_connected_callback self._on_close_cb = on_close_callback if sock: self.sock = sock self.sock.setblocking(False) self.watcher = kernel.loop.io(self.sock.fileno(), pyev.EV_READ | pyev.EV_WRITE, self._on_connecting) self.on_connected() else: sock_type = socket.SOCK_DGRAM if dgram else socket.SOCK_STREAM self.sock = socket.socket(socket.AF_UNIX, sock_type) self.sock.setblocking(False) ret = self.sock.connect_ex(endpoint) if ret != 0: raise IOError("Async connect to endpoint error: %s" % errorcode.get(ret)) self.watcher = kernel.loop.io(self.sock.fileno(), pyev.EV_READ | pyev.EV_WRITE, self._on_connecting) self.watcher.start() __handlers__.add(self)
def mkdir(self, handler, entry, path): abspath = self.storage_dispatch(entry, path, sd_only=True) try: os.mkdir(abspath) handler.send_text("ok") except OSError as e: raise RuntimeError("OSERR_" + errorcode.get(e.args[0], "UNKNOW"))
def check_rabbitmq_ports(result_handler): """ Checks all ports of Open vStorage components rabbitMQ and celery :param result_handler: logging object :type result_handler: ovs.extensions.healthcheck.result.HCResults :return: None :rtype: NoneType """ # Check Celery and RabbitMQ if OpenvStorageHealthCheck.LOCAL_SR.node_type != 'MASTER': result_handler.skip('RabbitMQ is not running/active on this server!') return result_handler.info('Checking Celery.', add_to_result=False) from errno import errorcode try: # noinspection PyUnresolvedReferences from celery.task.control import inspect stats = inspect().stats() if stats: result_handler.success('Successfully connected to Celery on all nodes.', code=ErrorCodes.port_celery) else: result_handler.failure('No running Celery workers were found.', code=ErrorCodes.port_celery) except IOError as ex: msg = 'Could not connect to Celery. Got {0}.'.format(ex) if len(ex.args) > 0 and errorcode.get(ex.args[0]) == 'ECONNREFUSED': msg += ' Check that the RabbitMQ server is running.' result_handler.failure(msg, code=ErrorCodes.port_celery) except ImportError as ex: result_handler.failure('Could not import the celery module. Got {}'.format(str(ex)), code=ErrorCodes.port_celery)
def check_celery_backend(self): """Checks if Celery backend is running and configured properly.""" print "Checking Celery Backend......", if 'celeryd' not in commands.getoutput('ps -ef'): self._set_status(0, "[%s]Error: celery is not running" % self.NAME) return True if not os.path.exists('/etc/compass/celeryconfig'): self._set_status( 0, "[%s]Error: No celery config file found for Compass" % self.NAME) return True try: insp = inspect() celery_stats = inspect.stats(insp) print celery_stats, except IOError as error: self._set_status( 0, "[%s]Error: Failed to connect to the backend: %s" % (self.NAME, str(error))) from errno import errorcode if ( len(error.args) > 0 and errorcode.get(error.args[0]) == 'ECONNREFUSED' ): self.messages.append( "[%s]Error: RabbitMQ server isn't running" % self.NAME) return True
def get_worker_status(): status_message = None try: status = WORKER_READY from celery.task.control import inspect insp = inspect() if insp.active(): status = WORKER_READY else: status = WORKER_OFFLINE status_message = "No running Celery workers were found." except IOError as e: from errno import errorcode status_message = "Error connecting to the backend: " + str(e) if len(e.args) > 0 and errorcode.get(e.args[0]) == 'ECONNREFUSED': status_message += ' Check that the RabbitMQ server is running.' status = WORKER_OFFLINE except ImportError as e: status = WORKER_ERROR status_message = e.message d = { 'status_code': status, 'status': get_worker_status_display(status) } if status_message: d['status_message'] = status_message return d
def rmfile(self, handler, entry, path): try: abspath = self.storage_dispatch(entry, path, sd_only=True, require_file=True) os.remove(abspath) handler.send_text("ok") except OSError as e: raise RuntimeError("OSERR_" + errorcode.get(e.args[0], "UNKNOW"))
def extractlayers(dc, args, layers, top_most_layer_id): target_path = args.target flags = O_WRONLY if target_path == _TARGET_STDOUT: target_fd = stdout.fileno() else: flags |= O_CREAT | O_TRUNC if not args.force: flags |= O_EXCL target_fd = logexception( _LOGGER, ERROR, 'unable to open target file "{}": {{e}}'.format(target_path), os_open, target_path, flags, 0o666) with fdopen(target_fd, 'wb') as target_file: if hasattr(target_file, 'seekable'): seekable = target_file.seekable() else: try: seekable = not lseek(target_fd, 0, SEEK_CUR) < 0 \ and S_ISREG(fstat(target_fd).st_mode) except OSError as e: if errorcode.get(e.errno) != 'ESPIPE': raise seekable = False open_args = {'fileobj': target_file} if args.compression is None: open_args['mode'] = 'w' if seekable else 'w|' else: if seekable: mode = 'w:{}' open_args['compresslevel'] = args.compress_level _, ext = ospath_splitext(target_path) if ext.lower() != '{}{}'.format(ospath_extsep, args.compression): _LOGGER.warning( 'target name "%s" doesn\'t match compression type ("%s")', target_path, args.compression) else: mode = 'w|{}' _LOGGER.warning( 'target "%s" is not seekable, ignoring compression level (%d)', target_path, args.compress_level) open_args['mode'] = mode.format(args.compression) with tarfile_open(**open_args) as tar_file: dimgx_extractlayers(dc, layers, tar_file, top_most_layer_id)
def cpfile(self, handler, from_entry, from_path, to_entry, to_path): try: abssource = self.storage_dispatch(from_entry, from_path, require_file=True) abstarget = self.storage_dispatch(to_entry, to_path, sd_only=True) shutil.copy(abssource, abstarget) handler.send_text("ok") except OSError as e: raise RuntimeError("OSERR_" + errorcode.get(e.args[0], "UNKNOW")) except ValueError as e: raise RuntimeError(BAD_PARAMS)
def download_file(self, handler, entry, path): def cb(h): handler.send_text("ok") try: abspath = self.storage_dispatch(entry, path, require_file=True) mimetype = mimetypes.guess_type(abspath)[0] length = os.path.getsize(abspath) stream = open(abspath, "rb") handler.async_send_binary(mimetype, length, stream, cb) except OSError as e: raise RuntimeError("OSERR_" + errorcode.get(e.args[0], "UNKNOW"))
def rabbitmq_is_running(): """ If checking for worker stats, an ``IOError`` may be raised depending on the problem for the RabbitMQ connection. """ try: celery_has_workers() except IOError as e: msg = "Error connecting to RabbitMQ: " + str(e) if len(e.args): if errorcode.get(e.args[0]) == 'ECONNREFUSED': msg = "RabbitMQ is not running or not reachable" raise SystemCheckError(msg)
def rabbitmq_is_running(): """ If checking for worker stats, an ``IOError`` may be raised depending on the problem for the RabbitMQ connection. """ try: celery_has_workers() except IOError as e: msg = "Error connecting to rabbitmq: " + str(e) if len(e.args): if errorcode.get(e.args[0]) == 'ECONNREFUSED': msg = "RabbitMQ is not running or not reachable" raise SystemCheckError(msg)
def send_network_config_request(request_data, before_send_callback=None): sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) sock.setblocking(False) ret = sock.connect_ex(NETWORK_MANAGE_ENDPOINT) if ret != 0: raise IOError("Async connect to endpoint error: %s" % errorcode.get(ret)) select((), (sock, ), (), 0.05) if before_send_callback: before_send_callback() sock.send(request_data) sock.close()
def lock_pidfile(pidfile): try: pid_handler = os.open(pidfile, os.O_CREAT | os.O_RDONLY | os.O_WRONLY, 0o644) select((), (pid_handler, ), (), 1.0) fcntl.lockf(pid_handler, fcntl.LOCK_EX | fcntl.LOCK_NB) return os.fdopen(pid_handler, "w") except IOError as e: if e.args[0] == EAGAIN: raise SystemError(0x80, 'Can not lock pidfile %s\n' % pidfile) else: raise SystemError( 0x81, 'Can not open pidfile %s (%s)\n' % (pidfile, errorcode.get(e.args[0], "?")))
def get_celery_worker_status(self): d = None try: insp = celery.task.control.inspect() if not insp.stats(): d = '没有找到可用的celery workers.' except IOError as e: msg = '无法连接celery backend: ' + str(e) if len(e.args) > 0 and errorcode.get(e.args[0]) == 'ECONNREFUSED': msg += '请检查RabbitMQ是否运行.' d = msg except ImportError as e: d = str(e) return d
def get_inspect_status(cls): try: from celery.task.control import inspect insp = inspect() d = insp.stats() if not d: d = {cls.ERROR_KEY: 'No running Celery workers were found.'} except IOError as e: from errno import errorcode msg = "Error connecting to the backend: " + str(e) if len(e.args) > 0 and errorcode.get(e.args[0]) == 'ECONNREFUSED': msg += ' Server connection refused.' d = {cls.ERROR_KEY: msg} except ImportError as e: d = {cls.ERROR_KEY: str(e)} return d
def _write(cls, tx, buf): # Low level send try: l = len(buf) ret = tx.write(buf[:512]) while ret < l: ret += tx.write(buf[ret:ret + 512]) except usb.core.USBError as e: if e.errno == ETIMEDOUT or e.backend_error_code == -116: raise FluxUSBError(*e.args, symbol=("TIMEOUT", )) else: logger.error("unhandle libusb error: %s", e) raise FluxUSBError(*e.args, symbol=("UNKNOWN_ERROR", errorcode.get(e.errno, e.errno)))
def md5(self, handler, entry, path): try: with open(self.storage_dispatch(entry, path, require_file=True), "rb") as f: buf = bytearray(4096) l = f.readinto(buf) m = md5() while l > 0: if l == 4096: m.update(buf) else: m.update(buf[:l]) l = f.readinto(buf) handler.send_text("md5 %s" % m.hexdigest()) except OSError as e: raise RuntimeError("OSERR_" + errorcode.get(e.args[0], "UNKNOW"))
def __init__(self, err, cmd=None, argdict=None, code=None): self.code = abs(code) if code is not None else None code_string = errorcode.get(self.code, str(self.code)) argdict = argdict if isinstance(argdict, dict) else {} if cmd is None and self.code is None: s = err elif cmd is None: s = 'error={} code={}'.format(err, code_string) else: cmd = cmd['prefix'] if isinstance( cmd, dict) and 'prefix' in cmd else cmd s = 'Executing "{} {}" failed: "{}" code={}'.format( cmd, ' '.join(['{}={}'.format(k, v) for k, v in argdict.items()]), err, code_string) super(ExternalCommandError, self).__init__(s)
def extractlayers(dc, args, layers, top_most_layer_id): target_path = args.target flags = O_WRONLY if target_path == _TARGET_STDOUT: target_fd = stdout.fileno() else: flags |= O_CREAT | O_TRUNC if not args.force: flags |= O_EXCL target_fd = logexception(_LOGGER, ERROR, 'unable to open target file "{}": {{e}}'.format(target_path), os_open, target_path, flags, 0o666) with fdopen(target_fd, 'wb') as target_file: if hasattr(target_file, 'seekable'): seekable = target_file.seekable() else: try: seekable = not lseek(target_fd, 0, SEEK_CUR) < 0 \ and S_ISREG(fstat(target_fd).st_mode) except OSError as e: if errorcode.get(e.errno) != 'ESPIPE': raise seekable = False open_args = { 'fileobj': target_file } if args.compression is None: open_args['mode'] = 'w' if seekable else 'w|' else: if seekable: mode = 'w:{}' open_args['compresslevel'] = args.compress_level _, ext = ospath_splitext(target_path) if ext.lower() != '{}{}'.format(ospath_extsep, args.compression): _LOGGER.warning('target name "%s" doesn\'t match compression type ("%s")', target_path, args.compression) else: mode = 'w|{}' _LOGGER.warning('target "%s" is not seekable, ignoring compression level (%d)', target_path, args.compress_level) open_args['mode'] = mode.format(args.compression) with tarfile_open(**open_args) as tar_file: dimgx_extractlayers(dc, layers, tar_file, top_most_layer_id)
def _send(self, buf): # Low level send try: l = len(buf) with self.tx_mutex: ret = self._tx.write(buf[:512]) while ret < l: ret += self._tx.write(buf[ret:ret + 512]) except usb.core.USBError as e: self._close_usbdev() if e.errno == ETIMEDOUT: raise FluxUSBError(*e.args, symbol=("TIMEOUT", )) else: raise FluxUSBError(*e.args, symbol=("UNKNOWN_ERROR", errorcode.get(e.errno, e.errno)))
def get_celery_worker_status(): ERROR_KEY = "ERROR" try: from celery.task.control import inspect insp = inspect() d = insp.stats() if not d: d = { ERROR_KEY: 'No running Celery workers were found.' } except IOError as e: from errno import errorcode msg = "Error connecting to the backend: " + str(e) if len(e.args) > 0 and errorcode.get(e.args[0]) == 'ECONNREFUSED': msg += ' Check that the RabbitMQ server is running.' d = { ERROR_KEY: msg } except ImportError as e: d = { ERROR_KEY: str(e)} return d
def _send(self, buf): # Low level send try: l = len(buf) with self.tx_mutex: ret = self._tx.write(buf[:512]) while ret < l: ret += self._tx.write(buf[ret:ret + 512]) except usb.core.USBError as e: self.close() if e.errno == ETIMEDOUT: raise FluxUSBError(*e.args, symbol=("TIMEOUT", )) else: raise FluxUSBError(*e.args, symbol=("UNKNOWN_ERROR", errorcode.get(e.errno, e.errno)))
def get_celery_worker_status(): ERROR_KEY = "ERROR" try: from celery.task.control import inspect insp = inspect() d = insp.stats() if not d: d = {ERROR_KEY: 'No running Celery workers were found.'} except IOError as e: from errno import errorcode msg = "Error connecting to the backend: " + str(e) if len(e.args) > 0 and errorcode.get(e.args[0]) == 'ECONNREFUSED': msg += ' Check that the RabbitMQ server is running.' d = {ERROR_KEY: msg} except ImportError as e: d = {ERROR_KEY: str(e)} return d
def get_celery_worker_status(): ERROR_KEY = "ERROR" try: insp = celeryapp.control.inspect() d = insp.stats() r = insp.registered() insp.active() if not d: d = {ERROR_KEY: 'No running Celery workers were found.'} except IOError as e: from errno import errorcode msg = "Error connecting to the brocker: " + str(e) if len(e.args) > 0 and errorcode.get(e.args[0]) == 'ECONNREFUSED': msg += ' Check that the RabbitMQ server is running.' d = {ERROR_KEY: msg} except ImportError as e: d = {ERROR_KEY: str(e)} return {"stats": d, "registered": r}
def get_celery_worker_status(): ERROR_KEY = "ERROR" try: from celery.task.control import inspect insp = inspect() d = insp.stats() if not d: d = False except IOError as e: from errno import errorcode msg = "Error connecting to the backend: " + str(e) if len(e.args) > 0 and errorcode.get(e.args[0]) == 'ECONNREFUSED': msg += ' Check that the Redis server is running.' print(msg) d = False except ImportError as e: d = False return d
def get_celery_worker_status(): ERROR_KEY = "ERROR" try: insp = celeryapp.control.inspect() d = insp.stats() r = insp.registered() insp.active() if not d: d = { ERROR_KEY: 'No running Celery workers were found.' } except IOError as e: from errno import errorcode msg = "Error connecting to the brocker: " + str(e) if len(e.args) > 0 and errorcode.get(e.args[0]) == 'ECONNREFUSED': msg += ' Check that the RabbitMQ server is running.' d = { ERROR_KEY: msg } except ImportError as e: d = { ERROR_KEY: str(e)} return {"stats" : d, "registered" : r}
def celery_worker_status(): """get celery worker status""" error_key = "ERROR" try: from celery.task.control import inspect insp = inspect() stats = insp.stats() if not stats: stats = {error_key: 'No running Celery workers were found.'} except IOError as err: from errno import errorcode msg = "Error connecting to the backend: " + str(err) if len(err.args) > 0 and errorcode.get(err.args[0]) == 'ECONNREFUSED': msg += ' Check that the RabbitMQ server is running.' stats = {error_key: msg} except ImportError as err: stats = {error_key: str(err)} return stats
def fetch_log(self, handler, path): filename = os.path.abspath( os.path.join("/var/db/fluxmonitord/run", path)) if filename.startswith("/var/db/fluxmonitord/run"): def cb(h): handler.send_text("ok") try: mimetype = mimetypes.guess_type(filename)[0] length = os.path.getsize(filename) stream = open(filename, "rb") handler.async_send_binary(mimetype or "binary", length, stream, cb) except OSError as e: raise RuntimeError( "OSERR_" + errorcode.get(e.args[0], "UNKNOW")) else: raise RuntimeError(BAD_PARAMS)
def get_celery_worker_status(): """ Get a dictionary of stats. If there is the ERROR_KEY, there is a problem and Jobs shouldn't run :return: """ try: from celery.task.control import inspect stats = inspect().stats() if not stats: stats = {ERROR_KEY: 'No running Celery workers were found.'} except IOError as e: from errno import errorcode msg = "Error connecting to the backend: " + str(e) if len(e.args) > 0 and errorcode.get(e.args[0]) == 'ECONNREFUSED': msg += ' Check that the RabbitMQ server is running.' stats = {ERROR_KEY: msg} except ImportError as e: stats = {ERROR_KEY: str(e)} return stats
def get_celery_worker_status(): ERROR_KEY = "ERROR" try: from celery import Celery app = Celery() app.config_from_object('celeryconf') insp = app.control.inspect() d = insp.stats() if not d: d = {ERROR_KEY: 'No running Celery workers were found.'} except IOError as e: from errno import errorcode msg = "Error connecting to the backend: " + str(e) if len(e.args) > 0 and errorcode.get(e.args[0]) == 'ECONNREFUSED': msg += ' Check the RabbitMQ server is running.' d = {ERROR_KEY: msg} except ImportError as e: d = {ERROR_KEY: str(e)} except Exception as x: d = {ERROR_KEY: str(x)} return d
def get_celery_status(): logger.info('Checking celery status ...') try: from celery.task.control import inspect stats = inspect().stats() if not stats: stats = {'ERROR': 'No running Celery workers were found.'} raise IOError('No celery worker found') except IOError as e: from errno import errorcode msg = "Error connecting to the backend: " + str(e) if len(e.args) > 0 and errorcode.get(e.args[0]) == 'ECONNREFUSED': msg += ' Check that the RabbitMQ server is running.' stats = {'ERROR': msg} logger.info(stats) raise (Exception(e)) except ImportError as e: stats = {'ERROR': str(e)} logger.info(stats) raise Exception(e) return stats
def get_celery_worker_status(): """ Detects if working """ # from # http://stackoverflow.com/questions/8506914/detect-whether-celery-is-available-running ERROR_KEY = "ERROR" try: from celery.task.control import inspect insp = inspect() d = insp.stats() if not d: d = {ERROR_KEY: 'No running Celery workers were found.'} except IOError as e: from errno import errorcode msg = "Error connecting to the backend: " + str(e) if len(e.args) > 0 and errorcode.get(e.args[0]) == 'ECONNREFUSED': msg += ' Check that the RabbitMQ server is running.' d = {ERROR_KEY: msg} except ImportError as e: d = {ERROR_KEY: str(e)} return d
def get_celery_worker_status(): """Checks whether celery is running and reports the error if not. Source: http://stackoverflow.com/questions/8506914/detect-whether-celery-is-available-running """ if hasattr(settings, 'BROKER_BACKEND') and settings.BROKER_BACKEND == 'memory': # We are testing with in-memory celery. Celery is effectively running. return {} try: insp = inspect() d = insp.stats() if not d: d = {CELERY_ERROR_KEY: 'No running Celery workers were found.'} except IOError as e: msg = "Error connecting to the backend: " + str(e) if len(e.args) > 0 and errorcode.get(e.args[0]) == 'ECONNREFUSED': msg += ' Check that the RabbitMQ server is running.' d = {CELERY_ERROR_KEY: msg} except ImportError as e: d = {CELERY_ERROR_KEY: str(e)} return d
def celery_check(analyzer): """ Function to check if Celery workers are up and running. """ try: from celery import Celery broker = analyzer.CELERY_BROKER_URL backend = analyzer.CELERY_RESULTS_BACKEND app = Celery('celery_tasks', broker=broker, backend=backend) app.config_from_object('celery_config') if not app.control.inspect().stats() and not app.control.inspect( ).ping(): raise Exception("Start celery workers. None running.") return True except IOError as e: msg = "Error connecting to the backend: " + str(e) from errno import errorcode if len(e.args) > 0 and errorcode.get(e.args[0]) == 'ECONNREFUSED': raise Exception("Check that the RabbitMQ server is running.") except ImportError as e: raise Exception("Celery module not available. Please install")
def perror(description): """Raises a runtime error from the specified description and ``errno``.""" errno = ffi.errno errname = errorcode.get(errno, str(errno)) return RuntimeError('{0}: errno = {1}'.format(description, errname))