Ejemplo n.º 1
0
    def __init__(self, shard):
        Service.__init__(self, shard)

        # Determine location of log file, and make directories.
        log_dir = os.path.join(config.log_dir, "cms")
        if not mkdir(config.log_dir) or \
                not mkdir(log_dir):
            logger.error("Cannot create necessary directories.")
            self.exit()
            return
        log_filename = "%d.log" % int(time.time())

        # Install a global file handler.
        self.file_handler = FileHandler(os.path.join(log_dir, log_filename),
                                        mode='w', encoding='utf-8')
        self.file_handler.setLevel(logging.DEBUG)
        self.file_handler.setFormatter(DetailedFormatter(False))
        root_logger.addHandler(self.file_handler)

        # Provide a symlink to the latest log file.
        try:
            os.remove(os.path.join(log_dir, "last.log"))
        except OSError:
            pass
        os.symlink(log_filename,
                   os.path.join(log_dir, "last.log"))

        self._last_messages = deque(maxlen=self.LAST_MESSAGES_COUNT)
Ejemplo n.º 2
0
    def __init__(self, shard):
        Service.__init__(self, shard)

        # Determine location of log file, and make directories.
        log_dir = os.path.join(config.log_dir, "cms")
        if not mkdir(config.log_dir) or \
                not mkdir(log_dir):
            logger.error("Cannot create necessary directories.")
            self.exit()
            return
        log_filename = "%d.log" % int(time.time())

        # Install a global file handler.
        self.file_handler = FileHandler(os.path.join(log_dir, log_filename),
                                        mode='w',
                                        encoding='utf-8')
        self.file_handler.setLevel(logging.DEBUG)
        self.file_handler.setFormatter(DetailedFormatter(False))
        root_logger.addHandler(self.file_handler)

        # Provide a symlink to the latest log file.
        try:
            os.remove(os.path.join(log_dir, "last.log"))
        except OSError:
            pass
        os.symlink(log_filename, os.path.join(log_dir, "last.log"))

        self._last_messages = deque(maxlen=self.LAST_MESSAGES_COUNT)
Ejemplo n.º 3
0
    def __init__(self, service=None, path=None):
        """Initialization.

        service (Service): the service we are running in. If None, we
                           simply avoid caching and allowing the
                           service to step in once in a while.
        path (string): if specified, back the FileCacher with a file
                       system-based storage instead that the default
                       database-based one. The specified directory
                       will be used as root for the storage and it
                       will be created if it doesn't exist.

        """
        self.service = service
        if path is None:
            self.backend = DBBackend(self.service)
        else:
            self.backend = FSBackend(path, self.service)
        if self.service is None:
            self.base_dir = tempfile.mkdtemp(dir=config.temp_dir)
        else:
            self.base_dir = os.path.join(
                config.cache_dir, "fs-cache-%s-%d" %
                (service._my_coord.name, service._my_coord.shard))
        self.tmp_dir = os.path.join(self.base_dir, "tmp")
        self.obj_dir = os.path.join(self.base_dir, "objects")
        if not mkdir(config.cache_dir) or \
               not mkdir(self.base_dir) or \
               not mkdir(self.tmp_dir) or \
               not mkdir(self.obj_dir):
            logger.error("Cannot create necessary directories.")
Ejemplo n.º 4
0
    def __init__(self, service=None, path=None):
        """Initialization.

        service (Service): the service we are running in. If None, we
                           simply avoid caching and allowing the
                           service to step in once in a while.
        path (string): if specified, back the FileCacher with a file
                       system-based storage instead that the default
                       database-based one. The specified directory
                       will be used as root for the storage and it
                       will be created if it doesn't exist.

        """
        self.service = service
        if path is None:
            self.backend = DBBackend(self.service)
        else:
            self.backend = FSBackend(path, self.service)
        if self.service is None:
            self.base_dir = tempfile.mkdtemp(dir=config.temp_dir)
        else:
            self.base_dir = os.path.join(
                config.cache_dir,
                "fs-cache-%s-%d" % (service._my_coord.name,
                                    service._my_coord.shard))
        self.tmp_dir = os.path.join(self.base_dir, "tmp")
        self.obj_dir = os.path.join(self.base_dir, "objects")
        if not mkdir(config.cache_dir) or \
               not mkdir(self.base_dir) or \
               not mkdir(self.tmp_dir) or \
               not mkdir(self.obj_dir):
            logger.error("Cannot create necessary directories.")
Ejemplo n.º 5
0
    def start_backdoor(self, backlog=50):
        """Start a backdoor server on a local UNIX domain socket.

        """
        backdoor_path = self.get_backdoor_path()
        try:
            os.remove(backdoor_path)
        except OSError as error:
            if error.errno != errno.ENOENT:
                raise
        else:
            logger.warning("A backdoor socket has been found and deleted.")
        mkdir(os.path.dirname(backdoor_path))
        backdoor_sock = gevent.socket.socket(socket.AF_UNIX,
                                             socket.SOCK_STREAM)
        backdoor_sock.setblocking(0)
        backdoor_sock.bind(backdoor_path)
        user = pwd.getpwnam(config.cmsuser)
        # We would like to also set the user to "cmsuser" but only root
        # can do that. Therefore we limit ourselves to the group.
        os.chown(backdoor_path, os.getuid(), user.pw_gid)
        os.chmod(backdoor_path, 0o770)
        backdoor_sock.listen(backlog)
        self.backdoor = BackdoorServer(backdoor_sock, locals={'service': self})
        self.backdoor.start()
Ejemplo n.º 6
0
    def purge_cache(self):
        """Empty the local cache.

        """
        self.destroy_cache()
        if not mkdir(config.cache_dir) or not mkdir(self.file_dir):
            logger.error("Cannot create necessary directories.")
            raise RuntimeError("Cannot create necessary directories.")
Ejemplo n.º 7
0
    def purge_cache(self):
        """Empty the local cache.

        """
        self.destroy_cache()
        if not mkdir(config.cache_dir) or not mkdir(self.file_dir):
            logger.error("Cannot create necessary directories.")
            raise RuntimeError("Cannot create necessary directories.")
Ejemplo n.º 8
0
    def purge_cache(self):
        """Delete all the content of the cache.

        """
        shutil.rmtree(self.base_dir)
        if not mkdir(config.cache_dir) or \
               not mkdir(self.base_dir) or \
               not mkdir(self.tmp_dir) or \
               not mkdir(self.obj_dir):
            logger.error("Cannot create necessary directories.")
Ejemplo n.º 9
0
    def purge_cache(self):
        """Delete all the content of the cache.

        """
        shutil.rmtree(self.base_dir)
        if not mkdir(config.cache_dir) or \
               not mkdir(self.base_dir) or \
               not mkdir(self.tmp_dir) or \
               not mkdir(self.obj_dir):
            logger.error("Cannot create necessary directories.")
Ejemplo n.º 10
0
    def purge_cache(self):
        """Empty the local cache.

        This function must not be called if the cache directory is shared.

        """
        self.destroy_cache()
        if not mkdir(config.cache_dir) or not mkdir(self.file_dir):
            logger.error("Cannot create necessary directories.")
            raise RuntimeError("Cannot create necessary directories.")
Ejemplo n.º 11
0
    def __init__(self, service=None, path=None, null=False):
        """Initialize.

        By default the database-powered backend will be used, but this
        can be changed using the parameters.

        service (Service|None): the service we are running for. Only
            used if present to determine the location of the
            file-system cache (and to provide the shard number to the
            Sandbox... sigh!).
        path (string|None): if specified, back the FileCacher with a
            file system-based storage instead of the default
            database-based one. The specified directory will be used
            as root for the storage and it will be created if it
            doesn't exist.
        null (bool): if True, back the FileCacher with a NullBackend,
            that just discards every file it receives. This setting
            takes priority over path.

        """
        self.service = service

        if null:
            self.backend = NullBackend()
        elif path is None:
            if config.s3_backend_enabled:
                self.backend = S3Backend(
                    region=config.s3_backend_region,
                    bucket=config.s3_backend_bucket,
                    prefix=config.s3_backend_prefix,
                    s3_proxy=config.s3_backend_proxy,
                    base_url_for_fetch=config.s3_backend_fetch_base_url,
                )
            else:
                self.backend = DBBackend()
        else:
            self.backend = FSBackend(path)

        if service is None:
            self.file_dir = tempfile.mkdtemp(dir=config.temp_dir)
            # Delete this directory on exit since it has a random name and
            # won't be used again.
            atexit.register(lambda: rmtree(self.file_dir))
        else:
            self.file_dir = os.path.join(
                config.cache_dir,
                "fs-cache-%s-%d" % (service.name, service.shard))

        self.temp_dir = os.path.join(self.file_dir, "_temp")

        if not mkdir(config.cache_dir) or not mkdir(config.temp_dir) \
                or not mkdir(self.file_dir) or not mkdir(self.temp_dir):
            logger.error("Cannot create necessary directories.")
            raise RuntimeError("Cannot create necessary directories.")
        atexit.register(lambda: rmtree(self.temp_dir))
Ejemplo n.º 12
0
    def initialize_logging(self):
        """Set up additional logging handlers.

        What we do, in detail, is to add a logger to file (whose
        filename depends on the coords) and a remote logger to a
        LogService. We also attach the service coords to all log
        messages.

        """
        filter_ = ServiceFilter(self.name, self.shard)

        # Update shell handler to attach service coords.
        shell_handler.addFilter(filter_)

        # Determine location of log file, and make directories.
        log_dir = os.path.join(config.log_dir,
                               "%s-%d" % (self.name, self.shard))
        mkdir(config.log_dir)
        mkdir(log_dir)

        log_filename = time.strftime("%Y-%m-%d-%H-%M-%S.log")

        # Install a file handler.
        file_handler = FileHandler(os.path.join(log_dir, log_filename),
                                   mode='w',
                                   encoding='utf-8')
        if config.file_log_debug:
            file_log_level = logging.DEBUG
        else:
            file_log_level = logging.INFO
        file_handler.setLevel(file_log_level)
        file_handler.setFormatter(DetailedFormatter(False))
        file_handler.addFilter(filter_)
        root_logger.addHandler(file_handler)

        # Provide a symlink to the latest log file.
        try:
            os.remove(os.path.join(log_dir, "last.log"))
        except OSError:
            pass
        os.symlink(log_filename, os.path.join(log_dir, "last.log"))

        # Setup a remote LogService handler (except when we already are
        # LogService, to avoid circular logging).
        if self.name != "LogService":
            log_service = self.connect_to(ServiceCoord("LogService", 0))
            remote_handler = LogServiceHandler(log_service)
            remote_handler.setLevel(logging.INFO)
            remote_handler.addFilter(filter_)
            root_logger.addHandler(remote_handler)
Ejemplo n.º 13
0
    def __init__(self, service=None, path=None, null=False, enabled=None):
        """Initialize.

        By default the database-powered backend will be used, but this
        can be changed using the parameters.

        service (Service|None): the service we are running for. Only
            used if present to determine the location of the
            file-system cache (and to provide the shard number to the
            Sandbox... sigh!).
        path (string|None): if specified, back the FileCacher with a
            file system-based storage instead of the default
            database-based one. The specified directory will be used
            as root for the storage and it will be created if it
            doesn't exist.
        null (bool): if True, back the FileCacher with a NullBackend,
            that just discards every file it receives. This setting
            takes priority over path.
        enabled (bool): overrides config.use_cache. If True, the files
            are always retrieved from the backend. Otherwise, the
            FileCacher behaves normally.

        """
        self.service = service

        if null:
            self.backend = NullBackend()
        elif path is None:
            self.backend = DBBackend()
        else:
            self.backend = FSBackend(path)

        if enabled is None:
            self.enabled = config.use_cache
        else:
            self.enabled = enabled

        if service is None:
            self.file_dir = tempfile.mkdtemp(dir=config.temp_dir)
        else:
            self.file_dir = os.path.join(
                config.cache_dir,
                "fs-cache-%s-%d" % (service.name, service.shard))

        self.temp_dir = os.path.join(self.file_dir, "_temp")

        if not mkdir(config.cache_dir) or not mkdir(self.file_dir) \
                or not mkdir(self.temp_dir):
            logger.error("Cannot create necessary directories.")
            raise RuntimeError("Cannot create necessary directories.")
Ejemplo n.º 14
0
    def __init__(self, service=None, path=None, null=False, enabled=None):
        """Initialize.

        By default the database-powered backend will be used, but this
        can be changed using the parameters.

        service (Service|None): the service we are running for. Only
            used if present to determine the location of the
            file-system cache (and to provide the shard number to the
            Sandbox... sigh!).
        path (string|None): if specified, back the FileCacher with a
            file system-based storage instead of the default
            database-based one. The specified directory will be used
            as root for the storage and it will be created if it
            doesn't exist.
        null (bool): if True, back the FileCacher with a NullBackend,
            that just discards every file it receives. This setting
            takes priority over path.
        enabled (bool): overrides config.use_cache. If True, the files
            are always retrieved from the backend. Otherwise, the
            FileCacher behaves normally.

        """
        self.service = service

        if null:
            self.backend = NullBackend()
        elif path is None:
            self.backend = DBBackend()
        else:
            self.backend = FSBackend(path)

        if enabled is None:
            self.enabled = config.use_cache
        else:
            self.enabled = enabled

        if service is None:
            self.file_dir = tempfile.mkdtemp(dir=config.temp_dir)
        else:
            self.file_dir = os.path.join(
                config.cache_dir,
                "fs-cache-%s-%d" % (service.name, service.shard))

        self.temp_dir = os.path.join(self.file_dir, "_temp")

        if not mkdir(config.cache_dir) or not mkdir(self.file_dir) \
                or not mkdir(self.temp_dir):
            logger.error("Cannot create necessary directories.")
            raise RuntimeError("Cannot create necessary directories.")
Ejemplo n.º 15
0
    def initialize_logging(self):
        """Set up additional logging handlers.

        What we do, in detail, is to add a logger to file (whose
        filename depends on the coords) and a remote logger to a
        LogService. We also attach the service coords to all log
        messages.

        """
        filter_ = ServiceFilter(self.name, self.shard)

        # Update shell handler to attach service coords.
        shell_handler.addFilter(filter_)

        # Determine location of log file, and make directories.
        log_dir = os.path.join(config.log_dir,
                               "%s-%d" % (self.name, self.shard))
        mkdir(config.log_dir)
        mkdir(log_dir)

        log_filename = time.strftime("%Y-%m-%d-%H-%M-%S.log")

        # Install a file handler.
        file_handler = FileHandler(os.path.join(log_dir, log_filename),
                                   mode='w', encoding='utf-8')
        if config.file_log_debug:
            file_log_level = logging.DEBUG
        else:
            file_log_level = logging.INFO
        file_handler.setLevel(file_log_level)
        file_handler.setFormatter(DetailedFormatter(False))
        file_handler.addFilter(filter_)
        root_logger.addHandler(file_handler)

        # Provide a symlink to the latest log file.
        try:
            os.remove(os.path.join(log_dir, "last.log"))
        except OSError:
            pass
        os.symlink(log_filename, os.path.join(log_dir, "last.log"))

        # Setup a remote LogService handler (except when we already are
        # LogService, to avoid circular logging).
        if self.name != "LogService":
            log_service = self.connect_to(ServiceCoord("LogService", 0))
            remote_handler = LogServiceHandler(log_service)
            remote_handler.setLevel(logging.INFO)
            remote_handler.addFilter(filter_)
            root_logger.addHandler(remote_handler)
Ejemplo n.º 16
0
def initialize_logging(service_name, service_shard):
    """Set up additional logging handlers.

    Some of the logging handlers are only activated when running for a
    service (by choice or because of technical issues). We therefore
    provide this method for services to call as soon as possible, with
    their coords as parameters, to complete logger initialization.

    What we do, in detail, is to add a logger to file (whose filename
    depends on the coords) and a remote logger to a LogService. We also
    attach the service coords to all log messages.

    """
    filter_ = ServiceFilter(service_name, service_shard)

    # Update shell handler to attach service coords.
    shell_handler.addFilter(filter_)

    # Determine location of log file, and make directories.
    log_dir = os.path.join(config.log_dir,
                           "%s-%d" % (service_name, service_shard))
    mkdir(config.log_dir)
    mkdir(log_dir)
    log_filename = "%d.log" % int(time.time())

    # Install a file handler.
    file_handler = FileHandler(os.path.join(log_dir, log_filename),
                               mode='w', encoding='utf-8')
    file_handler.setLevel(logging.INFO)
    file_handler.setFormatter(CustomFormatter(False))
    file_handler.addFilter(filter_)
    root_logger.addHandler(file_handler)

    # Provide a symlink to the latest log file.
    try:
        os.remove(os.path.join(log_dir, "last.log"))
    except OSError:
        pass
    os.symlink(log_filename,
               os.path.join(log_dir, "last.log"))

    # Setup a remote LogService handler (except when we already are
    # LogService, to avoid circular logging).
    if service_name != "LogService":
        remote_handler = LogServiceHandler()
        remote_handler.setLevel(logging.INFO)
        remote_handler.addFilter(filter_)
        root_logger.addHandler(remote_handler)
Ejemplo n.º 17
0
Archivo: log.py Proyecto: shaunren/cms
def initialize_logging(service_name, service_shard):
    """Set up additional logging handlers.

    Some of the logging handlers are only activated when running for a
    service (by choice or because of technical issues). We therefore
    provide this method for services to call as soon as possible, with
    their coords as parameters, to complete logger initialization.

    What we do, in detail, is to add a logger to file (whose filename
    depends on the coords) and a remote logger to a LogService. We also
    attach the service coords to all log messages.

    """
    filter_ = ServiceFilter(service_name, service_shard)

    # Update shell handler to attach service coords.
    shell_handler.addFilter(filter_)

    # Determine location of log file, and make directories.
    log_dir = os.path.join(config.log_dir,
                           "%s-%d" % (service_name, service_shard))
    mkdir(config.log_dir)
    mkdir(log_dir)
    log_filename = "%d.log" % int(time.time())

    # Install a file handler.
    file_handler = FileHandler(os.path.join(log_dir, log_filename),
                               mode='w',
                               encoding='utf-8')
    file_handler.setLevel(logging.DEBUG)
    file_handler.setFormatter(CustomFormatter(False))
    file_handler.addFilter(filter_)
    root_logger.addHandler(file_handler)

    # Provide a symlink to the latest log file.
    try:
        os.remove(os.path.join(log_dir, "last.log"))
    except OSError:
        pass
    os.symlink(log_filename, os.path.join(log_dir, "last.log"))

    # Setup a remote LogService handler (except when we already are
    # LogService, to avoid circular logging).
    if service_name != "LogService":
        remote_handler = LogServiceHandler()
        remote_handler.setLevel(logging.INFO)
        remote_handler.addFilter(filter_)
        root_logger.addHandler(remote_handler)
Ejemplo n.º 18
0
    def __init__(self, shard):
        logger.initialize(ServiceCoord("LogService", shard))
        Service.__init__(self, shard, custom_logger=logger)

        log_dir = os.path.join(config.log_dir, "cms")
        if not mkdir(config.log_dir) or \
               not mkdir(log_dir):
            logger.error("Cannot create necessary directories.")
            self.exit()
            return

        log_filename = "%d.log" % int(time.time())
        self._log_file = codecs.open(os.path.join(log_dir, log_filename), "w",
                                     "utf-8")
        try:
            os.remove(os.path.join(log_dir, "last.log"))
        except OSError:
            pass
        os.symlink(log_filename, os.path.join(log_dir, "last.log"))

        self._last_messages = []
Ejemplo n.º 19
0
    def __init__(self, shard):
        logger.initialize(ServiceCoord("LogService", shard))
        Service.__init__(self, shard, custom_logger=logger)

        log_dir = os.path.join(config.log_dir, "cms")
        if not mkdir(config.log_dir) or \
               not mkdir(log_dir):
            logger.error("Cannot create necessary directories.")
            self.exit()
            return

        log_filename = "%d.log" % int(time.time())
        self._log_file = codecs.open(os.path.join(log_dir, log_filename),
                                     "w", "utf-8")
        try:
            os.remove(os.path.join(log_dir, "last.log"))
        except OSError:
            pass
        os.symlink(log_filename,
                   os.path.join(log_dir, "last.log"))

        self._last_messages = []
Ejemplo n.º 20
0
    def __init__(self, service=None):
        """Initialization.

        service (Service): the service we are running in. If None, we
                           simply avoid caching and allowing the
                           service to step in once in a while.

        """
        self.service = service
        if self.service is None:
            self.base_dir = tempfile.mkdtemp(config.temp_dir)
        else:
            self.base_dir = os.path.join(
                config.cache_dir,
                "fs-cache-%s-%d" % (service._my_coord.name,
                                    service._my_coord.shard))
        self.tmp_dir = os.path.join(self.base_dir, "tmp")
        self.obj_dir = os.path.join(self.base_dir, "objects")
        if not mkdir(config.cache_dir) or \
               not mkdir(self.base_dir) or \
               not mkdir(self.tmp_dir) or \
               not mkdir(self.obj_dir):
            logger.error("Cannot create necessary directories.")
Ejemplo n.º 21
0
    def start_backdoor(self, backlog=50):
        """Start a backdoor server on a local UNIX domain socket.

        """
        backdoor_path = self.get_backdoor_path()
        try:
            os.remove(backdoor_path)
        except OSError as error:
            if error.errno != errno.ENOENT:
                raise
        else:
            logger.warning("A backdoor socket has been found and deleted.")
        mkdir(os.path.dirname(backdoor_path))
        backdoor_sock = _socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
        backdoor_sock.setblocking(0)
        backdoor_sock.bind(backdoor_path)
        user = pwd.getpwnam("cmsuser")
        # We would like to also set the user to "cmsuser" but only root
        # can do that. Therefore we limit ourselves to the group.
        os.chown(backdoor_path, os.getuid(), user.pw_gid)
        os.chmod(backdoor_path, 0o770)
        backdoor_sock.listen(backlog)
        self.backdoor = BackdoorServer(backdoor_sock, locals={'service': self})
        self.backdoor.start()
Ejemplo n.º 22
0
 def _create_directory_or_die(self, dir):
     """Create dir and ensure it exists, or raise a RuntimeError."""
     if not mkdir(dir):
         msg = "Cannot create required directory '%s'." % dir
         logger.error(msg)
         raise RuntimeError(msg)
Ejemplo n.º 23
0
 def _create_directory_or_die(directory):
     """Create directory and ensure it exists, or raise a RuntimeError."""
     if not mkdir(directory):
         msg = "Cannot create required directory '%s'." % directory
         logger.error(msg)
         raise RuntimeError(msg)