def __init__(self, shard):
Service.__init__(self, shard)
# Determine location of log file, and make directories.
log_dir = os.path.join(config.log_dir, "cms")
if not mkdir(config.log_dir) or \
not mkdir(log_dir):
logger.error("Cannot create necessary directories.")
self.exit()
return
log_filename = "%d.log" % int(time.time())
# Install a global file handler.
self.file_handler = FileHandler(os.path.join(log_dir, log_filename),
mode='w', encoding='utf-8')
self.file_handler.setLevel(logging.DEBUG)
self.file_handler.setFormatter(DetailedFormatter(False))
root_logger.addHandler(self.file_handler)
# Provide a symlink to the latest log file.
try:
os.remove(os.path.join(log_dir, "last.log"))
except OSError:
pass
os.symlink(log_filename,
os.path.join(log_dir, "last.log"))
self._last_messages = deque(maxlen=self.LAST_MESSAGES_COUNT)
def __init__(self, shard):
Service.__init__(self, shard)
# Determine location of log file, and make directories.
log_dir = os.path.join(config.log_dir, "cms")
if not mkdir(config.log_dir) or \
not mkdir(log_dir):
logger.error("Cannot create necessary directories.")
self.exit()
return
log_filename = "%d.log" % int(time.time())
# Install a global file handler.
self.file_handler = FileHandler(os.path.join(log_dir, log_filename),
mode='w',
encoding='utf-8')
self.file_handler.setLevel(logging.DEBUG)
self.file_handler.setFormatter(DetailedFormatter(False))
root_logger.addHandler(self.file_handler)
# Provide a symlink to the latest log file.
try:
os.remove(os.path.join(log_dir, "last.log"))
except OSError:
pass
os.symlink(log_filename, os.path.join(log_dir, "last.log"))
self._last_messages = deque(maxlen=self.LAST_MESSAGES_COUNT)
def __init__(self, service=None, path=None):
"""Initialization.
service (Service): the service we are running in. If None, we
simply avoid caching and allowing the
service to step in once in a while.
path (string): if specified, back the FileCacher with a file
system-based storage instead that the default
database-based one. The specified directory
will be used as root for the storage and it
will be created if it doesn't exist.
"""
self.service = service
if path is None:
self.backend = DBBackend(self.service)
else:
self.backend = FSBackend(path, self.service)
if self.service is None:
self.base_dir = tempfile.mkdtemp(dir=config.temp_dir)
else:
self.base_dir = os.path.join(
config.cache_dir, "fs-cache-%s-%d" %
(service._my_coord.name, service._my_coord.shard))
self.tmp_dir = os.path.join(self.base_dir, "tmp")
self.obj_dir = os.path.join(self.base_dir, "objects")
if not mkdir(config.cache_dir) or \
not mkdir(self.base_dir) or \
not mkdir(self.tmp_dir) or \
not mkdir(self.obj_dir):
logger.error("Cannot create necessary directories.")
def __init__(self, service=None, path=None):
"""Initialization.
service (Service): the service we are running in. If None, we
simply avoid caching and allowing the
service to step in once in a while.
path (string): if specified, back the FileCacher with a file
system-based storage instead that the default
database-based one. The specified directory
will be used as root for the storage and it
will be created if it doesn't exist.
"""
self.service = service
if path is None:
self.backend = DBBackend(self.service)
else:
self.backend = FSBackend(path, self.service)
if self.service is None:
self.base_dir = tempfile.mkdtemp(dir=config.temp_dir)
else:
self.base_dir = os.path.join(
config.cache_dir,
"fs-cache-%s-%d" % (service._my_coord.name,
service._my_coord.shard))
self.tmp_dir = os.path.join(self.base_dir, "tmp")
self.obj_dir = os.path.join(self.base_dir, "objects")
if not mkdir(config.cache_dir) or \
not mkdir(self.base_dir) or \
not mkdir(self.tmp_dir) or \
not mkdir(self.obj_dir):
logger.error("Cannot create necessary directories.")
def start_backdoor(self, backlog=50):
"""Start a backdoor server on a local UNIX domain socket.
"""
backdoor_path = self.get_backdoor_path()
try:
os.remove(backdoor_path)
except OSError as error:
if error.errno != errno.ENOENT:
raise
else:
logger.warning("A backdoor socket has been found and deleted.")
mkdir(os.path.dirname(backdoor_path))
backdoor_sock = gevent.socket.socket(socket.AF_UNIX,
socket.SOCK_STREAM)
backdoor_sock.setblocking(0)
backdoor_sock.bind(backdoor_path)
user = pwd.getpwnam(config.cmsuser)
# We would like to also set the user to "cmsuser" but only root
# can do that. Therefore we limit ourselves to the group.
os.chown(backdoor_path, os.getuid(), user.pw_gid)
os.chmod(backdoor_path, 0o770)
backdoor_sock.listen(backlog)
self.backdoor = BackdoorServer(backdoor_sock, locals={'service': self})
self.backdoor.start()
def purge_cache(self):
"""Empty the local cache.
"""
self.destroy_cache()
if not mkdir(config.cache_dir) or not mkdir(self.file_dir):
logger.error("Cannot create necessary directories.")
raise RuntimeError("Cannot create necessary directories.")
def purge_cache(self):
"""Empty the local cache.
"""
self.destroy_cache()
if not mkdir(config.cache_dir) or not mkdir(self.file_dir):
logger.error("Cannot create necessary directories.")
raise RuntimeError("Cannot create necessary directories.")
def purge_cache(self):
"""Delete all the content of the cache.
"""
shutil.rmtree(self.base_dir)
if not mkdir(config.cache_dir) or \
not mkdir(self.base_dir) or \
not mkdir(self.tmp_dir) or \
not mkdir(self.obj_dir):
logger.error("Cannot create necessary directories.")
def purge_cache(self):
"""Delete all the content of the cache.
"""
shutil.rmtree(self.base_dir)
if not mkdir(config.cache_dir) or \
not mkdir(self.base_dir) or \
not mkdir(self.tmp_dir) or \
not mkdir(self.obj_dir):
logger.error("Cannot create necessary directories.")
def purge_cache(self):
"""Empty the local cache.
This function must not be called if the cache directory is shared.
"""
self.destroy_cache()
if not mkdir(config.cache_dir) or not mkdir(self.file_dir):
logger.error("Cannot create necessary directories.")
raise RuntimeError("Cannot create necessary directories.")
def __init__(self, service=None, path=None, null=False):
"""Initialize.
By default the database-powered backend will be used, but this
can be changed using the parameters.
service (Service|None): the service we are running for. Only
used if present to determine the location of the
file-system cache (and to provide the shard number to the
Sandbox... sigh!).
path (string|None): if specified, back the FileCacher with a
file system-based storage instead of the default
database-based one. The specified directory will be used
as root for the storage and it will be created if it
doesn't exist.
null (bool): if True, back the FileCacher with a NullBackend,
that just discards every file it receives. This setting
takes priority over path.
"""
self.service = service
if null:
self.backend = NullBackend()
elif path is None:
if config.s3_backend_enabled:
self.backend = S3Backend(
region=config.s3_backend_region,
bucket=config.s3_backend_bucket,
prefix=config.s3_backend_prefix,
s3_proxy=config.s3_backend_proxy,
base_url_for_fetch=config.s3_backend_fetch_base_url,
)
else:
self.backend = DBBackend()
else:
self.backend = FSBackend(path)
if service is None:
self.file_dir = tempfile.mkdtemp(dir=config.temp_dir)
# Delete this directory on exit since it has a random name and
# won't be used again.
atexit.register(lambda: rmtree(self.file_dir))
else:
self.file_dir = os.path.join(
config.cache_dir,
"fs-cache-%s-%d" % (service.name, service.shard))
self.temp_dir = os.path.join(self.file_dir, "_temp")
if not mkdir(config.cache_dir) or not mkdir(config.temp_dir) \
or not mkdir(self.file_dir) or not mkdir(self.temp_dir):
logger.error("Cannot create necessary directories.")
raise RuntimeError("Cannot create necessary directories.")
atexit.register(lambda: rmtree(self.temp_dir))
def initialize_logging(self):
"""Set up additional logging handlers.
What we do, in detail, is to add a logger to file (whose
filename depends on the coords) and a remote logger to a
LogService. We also attach the service coords to all log
messages.
"""
filter_ = ServiceFilter(self.name, self.shard)
# Update shell handler to attach service coords.
shell_handler.addFilter(filter_)
# Determine location of log file, and make directories.
log_dir = os.path.join(config.log_dir,
"%s-%d" % (self.name, self.shard))
mkdir(config.log_dir)
mkdir(log_dir)
log_filename = time.strftime("%Y-%m-%d-%H-%M-%S.log")
# Install a file handler.
file_handler = FileHandler(os.path.join(log_dir, log_filename),
mode='w',
encoding='utf-8')
if config.file_log_debug:
file_log_level = logging.DEBUG
else:
file_log_level = logging.INFO
file_handler.setLevel(file_log_level)
file_handler.setFormatter(DetailedFormatter(False))
file_handler.addFilter(filter_)
root_logger.addHandler(file_handler)
# Provide a symlink to the latest log file.
try:
os.remove(os.path.join(log_dir, "last.log"))
except OSError:
pass
os.symlink(log_filename, os.path.join(log_dir, "last.log"))
# Setup a remote LogService handler (except when we already are
# LogService, to avoid circular logging).
if self.name != "LogService":
log_service = self.connect_to(ServiceCoord("LogService", 0))
remote_handler = LogServiceHandler(log_service)
remote_handler.setLevel(logging.INFO)
remote_handler.addFilter(filter_)
root_logger.addHandler(remote_handler)
def __init__(self, service=None, path=None, null=False, enabled=None):
"""Initialize.
By default the database-powered backend will be used, but this
can be changed using the parameters.
service (Service|None): the service we are running for. Only
used if present to determine the location of the
file-system cache (and to provide the shard number to the
Sandbox... sigh!).
path (string|None): if specified, back the FileCacher with a
file system-based storage instead of the default
database-based one. The specified directory will be used
as root for the storage and it will be created if it
doesn't exist.
null (bool): if True, back the FileCacher with a NullBackend,
that just discards every file it receives. This setting
takes priority over path.
enabled (bool): overrides config.use_cache. If True, the files
are always retrieved from the backend. Otherwise, the
FileCacher behaves normally.
"""
self.service = service
if null:
self.backend = NullBackend()
elif path is None:
self.backend = DBBackend()
else:
self.backend = FSBackend(path)
if enabled is None:
self.enabled = config.use_cache
else:
self.enabled = enabled
if service is None:
self.file_dir = tempfile.mkdtemp(dir=config.temp_dir)
else:
self.file_dir = os.path.join(
config.cache_dir,
"fs-cache-%s-%d" % (service.name, service.shard))
self.temp_dir = os.path.join(self.file_dir, "_temp")
if not mkdir(config.cache_dir) or not mkdir(self.file_dir) \
or not mkdir(self.temp_dir):
logger.error("Cannot create necessary directories.")
raise RuntimeError("Cannot create necessary directories.")
def __init__(self, service=None, path=None, null=False, enabled=None):
"""Initialize.
By default the database-powered backend will be used, but this
can be changed using the parameters.
service (Service|None): the service we are running for. Only
used if present to determine the location of the
file-system cache (and to provide the shard number to the
Sandbox... sigh!).
path (string|None): if specified, back the FileCacher with a
file system-based storage instead of the default
database-based one. The specified directory will be used
as root for the storage and it will be created if it
doesn't exist.
null (bool): if True, back the FileCacher with a NullBackend,
that just discards every file it receives. This setting
takes priority over path.
enabled (bool): overrides config.use_cache. If True, the files
are always retrieved from the backend. Otherwise, the
FileCacher behaves normally.
"""
self.service = service
if null:
self.backend = NullBackend()
elif path is None:
self.backend = DBBackend()
else:
self.backend = FSBackend(path)
if enabled is None:
self.enabled = config.use_cache
else:
self.enabled = enabled
if service is None:
self.file_dir = tempfile.mkdtemp(dir=config.temp_dir)
else:
self.file_dir = os.path.join(
config.cache_dir,
"fs-cache-%s-%d" % (service.name, service.shard))
self.temp_dir = os.path.join(self.file_dir, "_temp")
if not mkdir(config.cache_dir) or not mkdir(self.file_dir) \
or not mkdir(self.temp_dir):
logger.error("Cannot create necessary directories.")
raise RuntimeError("Cannot create necessary directories.")
def initialize_logging(self):
"""Set up additional logging handlers.
What we do, in detail, is to add a logger to file (whose
filename depends on the coords) and a remote logger to a
LogService. We also attach the service coords to all log
messages.
"""
filter_ = ServiceFilter(self.name, self.shard)
# Update shell handler to attach service coords.
shell_handler.addFilter(filter_)
# Determine location of log file, and make directories.
log_dir = os.path.join(config.log_dir,
"%s-%d" % (self.name, self.shard))
mkdir(config.log_dir)
mkdir(log_dir)
log_filename = time.strftime("%Y-%m-%d-%H-%M-%S.log")
# Install a file handler.
file_handler = FileHandler(os.path.join(log_dir, log_filename),
mode='w', encoding='utf-8')
if config.file_log_debug:
file_log_level = logging.DEBUG
else:
file_log_level = logging.INFO
file_handler.setLevel(file_log_level)
file_handler.setFormatter(DetailedFormatter(False))
file_handler.addFilter(filter_)
root_logger.addHandler(file_handler)
# Provide a symlink to the latest log file.
try:
os.remove(os.path.join(log_dir, "last.log"))
except OSError:
pass
os.symlink(log_filename, os.path.join(log_dir, "last.log"))
# Setup a remote LogService handler (except when we already are
# LogService, to avoid circular logging).
if self.name != "LogService":
log_service = self.connect_to(ServiceCoord("LogService", 0))
remote_handler = LogServiceHandler(log_service)
remote_handler.setLevel(logging.INFO)
remote_handler.addFilter(filter_)
root_logger.addHandler(remote_handler)
def initialize_logging(service_name, service_shard):
"""Set up additional logging handlers.
Some of the logging handlers are only activated when running for a
service (by choice or because of technical issues). We therefore
provide this method for services to call as soon as possible, with
their coords as parameters, to complete logger initialization.
What we do, in detail, is to add a logger to file (whose filename
depends on the coords) and a remote logger to a LogService. We also
attach the service coords to all log messages.
"""
filter_ = ServiceFilter(service_name, service_shard)
# Update shell handler to attach service coords.
shell_handler.addFilter(filter_)
# Determine location of log file, and make directories.
log_dir = os.path.join(config.log_dir,
"%s-%d" % (service_name, service_shard))
mkdir(config.log_dir)
mkdir(log_dir)
log_filename = "%d.log" % int(time.time())
# Install a file handler.
file_handler = FileHandler(os.path.join(log_dir, log_filename),
mode='w', encoding='utf-8')
file_handler.setLevel(logging.INFO)
file_handler.setFormatter(CustomFormatter(False))
file_handler.addFilter(filter_)
root_logger.addHandler(file_handler)
# Provide a symlink to the latest log file.
try:
os.remove(os.path.join(log_dir, "last.log"))
except OSError:
pass
os.symlink(log_filename,
os.path.join(log_dir, "last.log"))
# Setup a remote LogService handler (except when we already are
# LogService, to avoid circular logging).
if service_name != "LogService":
remote_handler = LogServiceHandler()
remote_handler.setLevel(logging.INFO)
remote_handler.addFilter(filter_)
root_logger.addHandler(remote_handler)
def initialize_logging(service_name, service_shard):
"""Set up additional logging handlers.
Some of the logging handlers are only activated when running for a
service (by choice or because of technical issues). We therefore
provide this method for services to call as soon as possible, with
their coords as parameters, to complete logger initialization.
What we do, in detail, is to add a logger to file (whose filename
depends on the coords) and a remote logger to a LogService. We also
attach the service coords to all log messages.
"""
filter_ = ServiceFilter(service_name, service_shard)
# Update shell handler to attach service coords.
shell_handler.addFilter(filter_)
# Determine location of log file, and make directories.
log_dir = os.path.join(config.log_dir,
"%s-%d" % (service_name, service_shard))
mkdir(config.log_dir)
mkdir(log_dir)
log_filename = "%d.log" % int(time.time())
# Install a file handler.
file_handler = FileHandler(os.path.join(log_dir, log_filename),
mode='w',
encoding='utf-8')
file_handler.setLevel(logging.DEBUG)
file_handler.setFormatter(CustomFormatter(False))
file_handler.addFilter(filter_)
root_logger.addHandler(file_handler)
# Provide a symlink to the latest log file.
try:
os.remove(os.path.join(log_dir, "last.log"))
except OSError:
pass
os.symlink(log_filename, os.path.join(log_dir, "last.log"))
# Setup a remote LogService handler (except when we already are
# LogService, to avoid circular logging).
if service_name != "LogService":
remote_handler = LogServiceHandler()
remote_handler.setLevel(logging.INFO)
remote_handler.addFilter(filter_)
root_logger.addHandler(remote_handler)
def __init__(self, shard):
logger.initialize(ServiceCoord("LogService", shard))
Service.__init__(self, shard, custom_logger=logger)
log_dir = os.path.join(config.log_dir, "cms")
if not mkdir(config.log_dir) or \
not mkdir(log_dir):
logger.error("Cannot create necessary directories.")
self.exit()
return
log_filename = "%d.log" % int(time.time())
self._log_file = codecs.open(os.path.join(log_dir, log_filename), "w",
"utf-8")
try:
os.remove(os.path.join(log_dir, "last.log"))
except OSError:
pass
os.symlink(log_filename, os.path.join(log_dir, "last.log"))
self._last_messages = []
def __init__(self, shard):
logger.initialize(ServiceCoord("LogService", shard))
Service.__init__(self, shard, custom_logger=logger)
log_dir = os.path.join(config.log_dir, "cms")
if not mkdir(config.log_dir) or \
not mkdir(log_dir):
logger.error("Cannot create necessary directories.")
self.exit()
return
log_filename = "%d.log" % int(time.time())
self._log_file = codecs.open(os.path.join(log_dir, log_filename),
"w", "utf-8")
try:
os.remove(os.path.join(log_dir, "last.log"))
except OSError:
pass
os.symlink(log_filename,
os.path.join(log_dir, "last.log"))
self._last_messages = []
def __init__(self, service=None):
"""Initialization.
service (Service): the service we are running in. If None, we
simply avoid caching and allowing the
service to step in once in a while.
"""
self.service = service
if self.service is None:
self.base_dir = tempfile.mkdtemp(config.temp_dir)
else:
self.base_dir = os.path.join(
config.cache_dir,
"fs-cache-%s-%d" % (service._my_coord.name,
service._my_coord.shard))
self.tmp_dir = os.path.join(self.base_dir, "tmp")
self.obj_dir = os.path.join(self.base_dir, "objects")
if not mkdir(config.cache_dir) or \
not mkdir(self.base_dir) or \
not mkdir(self.tmp_dir) or \
not mkdir(self.obj_dir):
logger.error("Cannot create necessary directories.")
def start_backdoor(self, backlog=50):
"""Start a backdoor server on a local UNIX domain socket.
"""
backdoor_path = self.get_backdoor_path()
try:
os.remove(backdoor_path)
except OSError as error:
if error.errno != errno.ENOENT:
raise
else:
logger.warning("A backdoor socket has been found and deleted.")
mkdir(os.path.dirname(backdoor_path))
backdoor_sock = _socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
backdoor_sock.setblocking(0)
backdoor_sock.bind(backdoor_path)
user = pwd.getpwnam("cmsuser")
# We would like to also set the user to "cmsuser" but only root
# can do that. Therefore we limit ourselves to the group.
os.chown(backdoor_path, os.getuid(), user.pw_gid)
os.chmod(backdoor_path, 0o770)
backdoor_sock.listen(backlog)
self.backdoor = BackdoorServer(backdoor_sock, locals={'service': self})
self.backdoor.start()
def _create_directory_or_die(self, dir):
"""Create dir and ensure it exists, or raise a RuntimeError."""
if not mkdir(dir):
msg = "Cannot create required directory '%s'." % dir
logger.error(msg)
raise RuntimeError(msg)
def _create_directory_or_die(directory):
"""Create directory and ensure it exists, or raise a RuntimeError."""
if not mkdir(directory):
msg = "Cannot create required directory '%s'." % directory
logger.error(msg)
raise RuntimeError(msg)
