def check_dir(self):
for h in list(range(48, 58)) + list(range(97, 103)):
for i in list(range(48, 58)) + list(range(97, 103)):
for j in list(range(48, 58)) + list(range(97, 103)):
initial = "0" + chr(h) + "/" + chr(i) + chr(j)
pathtool.mkdir(os.path.join(self.path, initial))
self.load_cache(initial)
def __init__(self, media, path):
self.media = type(media) is list and media or [media]
self.path = path
if self.path:
pathtool.mkdir(path)
self.logger_factory = {}
self.lock = Lock()
self.make_logger("server", "monthly")
self.make_logger("app", "daily")
self.make_logger("request", "daily")
def setup(self):
self.bind_signal(hTERM, hHUP)
val = self.config.get("max_retry", 10)
if val: self.MAX_RETRY = val
val = self.config.get("keep_days", 3)
if val: self.UNDELIVERS_KEEP_MAX = val * 3600 * 24
if self.config.get("smtpserver"):
self.log(
"using default SMTP: {}".format(self.config.get("smtpserver")),
"info")
else:
self.log("no default SMTP server provided", "warn")
self.default_smtp = (self.config.get("smtpserver"),
self.config.get("user"),
self.config.get("password"),
self.config.get("ssl") and True or False)
# dummy file like object for keeping lifetime loop
if os.name == "nt":
select_trigger.trigger.address = ('127.9.9.9', 19998)
select_trigger.trigger()
self.path_spool = os.path.join(self.varpath, "spool")
self.log("mail spooled in {}".format(self.path_spool), "info")
pathtool.mkdir(self.path_spool)
composer.Composer.SAVE_PATH = self.path_spool
self.path_undeliver = os.path.join(self.varpath, "undeliver")
self.log("undelivered mail saved in {}".format(self.path_undeliver),
"info")
pathtool.mkdir(self.path_undeliver)
lifetime.init(logger=self.logger)
lifetime.maintern.sched(2.0, self.handle_spool)
if os.name == "nt":
lifetime.maintern.sched(2.0, self.maintern_shutdown_request)
def run(**conf):
import os, sys, time
from . import Skitai
from rs4.psutil import flock
from rs4 import pathtool
import getopt
class SkitaiServer(Skitai.Loader):
NAME = 'instance'
def __init__(self, conf):
self.conf = conf
self.flock = None
Skitai.Loader.__init__(self, 'config', conf.get('logpath'),
conf.get('varpath'), conf.get("wasc"))
def close(self):
if self.wasc.httpserver.worker_ident == "master":
pass
Skitai.Loader.close(self)
def config_logger(self, path):
media = []
if path is not None:
media.append("file")
if self.conf.get('verbose', "no") in ("yes", "1", 1):
media.append("screen")
self.conf['verbose'] = "yes"
if not media:
media.append("screen")
self.conf['verbose'] = "yes"
Skitai.Loader.config_logger(self, path, media,
self.conf["log_off"])
def master_jobs(self):
self.wasc.logger("server",
"[info] engine tmp path: %s" % self.varpath)
if self.logpath:
self.wasc.logger("server",
"[info] engine log path: %s" % self.logpath)
self.conf.get("models-keys") and self.set_model_keys(
self.conf["models-keys"])
self.conf.get("states-keys") and self.set_state_keys(
self.conf["states-keys"])
def maintern_shutdown_request(self, now):
req = self.flock.lockread("signal")
if not req: return
self.wasc.logger("server", "[info] got signal - %s" % req)
if req == "terminate":
lifetime.shutdown(0, 30.0)
elif req == "restart":
lifetime.shutdown(3, 30.0)
elif req == "kill":
lifetime.shutdown(0, 1.0)
elif req == "rotate":
self.wasc.logger.rotate()
else:
self.wasc.logger("server", "[error] unknown signal - %s" % req)
self.flock.unlock("signal")
def configure(self):
conf = self.conf
self.set_num_worker(conf.get('workers', 1))
if conf.get("certfile"):
self.config_certification(conf.get("certfile"),
conf.get("keyfile"),
conf.get("passphrase"))
self.config_dns(dconf['dns_protocol'])
if conf.get("cachefs_diskmax", 0) and not conf.get("cachefs_dir"):
conf["cachefs_dir"] = os.path.join(self.varpath, "cachefs")
self.config_cachefs(conf.get("cachefs_dir", None),
conf.get("cachefs_memmax", 0),
conf.get("cachefs_diskmax", 0))
self.config_rcache(conf.get("rcache_objmax", 100))
if conf.get('fws_to'):
self.config_forward_server(conf.get('fws_address', '0.0.0.0'),
conf.get('fws_port', 80),
conf.get('fws_to', 443))
if "---port" in sys.argv:
port = int(sys.argv[sys.argv.index("---port") + 1])
else:
port = conf.get('port', 5000)
self.config_webserver(port,
conf.get('address', '0.0.0.0'),
NAME,
conf.get("certfile") is not None,
conf.get('keep_alive', 2),
conf.get('network_timeout', 10),
thunks=[self.master_jobs])
if os.name == "posix" and self.wasc.httpserver.worker_ident == "master":
# master does not serve
return
self.config_threads(conf.get('threads', 4))
self.config_backends(conf.get('backend_keep_alive', 1200))
for name, args in conf.get("clusters", {}).items():
ctype, members, policy, ssl = args
self.add_cluster(ctype, name, members, ssl, policy)
self.install_handler(
conf.get("mount"),
conf.get("proxy", False),
conf.get("max_ages", {}),
conf.get("blacklist_dir"), # blacklist_dir
conf.get("proxy_unsecure_https",
False), # disable unsecure https
conf.get("enable_gw", False), # API gateway
conf.get("gw_auth", False),
conf.get("gw_realm", "API Gateway"),
conf.get("gw_secret_key", None))
lifetime.init(logger=self.wasc.logger.get("server"))
lifetime.maintern.sched(3.0, dns.pool.maintern)
if os.name == "nt":
lifetime.maintern.sched(11.0, self.maintern_shutdown_request)
self.flock = flock.Lock(
os.path.join(self.varpath, ".%s" % self.NAME))
#----------------------------------------------------------------------------
global dconf, PROCESS_NAME, Win32Service
for k, v in dconf.items():
if k not in conf:
conf[k] = v
if "---production" in sys.argv:
os.environ["SKITAI_ENV"] = "PRODUCTION"
if conf.get("name"):
PROCESS_NAME = 'skitai/{}'.format(conf["name"])
if not conf.get('mount'):
raise systemError('No mount point')
conf["varpath"] = get_varpath(get_proc_title())
pathtool.mkdir(conf["varpath"])
if "logpath" in conf and not conf["logpath"]:
conf["logpath"] = get_logpath(get_proc_title())
cmd = get_command()
working_dir = getswd()
lockpath = conf["varpath"]
servicer = service.Service(get_proc_title(), working_dir, lockpath,
Win32Service)
if cmd and not servicer.execute(cmd):
return
if not cmd:
if servicer.status(False):
raise SystemError("daemon is running")
conf['verbose'] = 'yes'
elif cmd in ("start", "restart"):
sys.stderr = open(os.path.join(conf.get('varpath'), "stderr.engine"),
"a")
server = SkitaiServer(conf)
# timeout for fast keyboard interrupt on win32
try:
try:
server.run(conf.get('verbose') and 1.0 or 30.0)
except KeyboardInterrupt:
pass
finally:
_exit_code = server.get_exit_code()
if _exit_code is not None: # master process
sys.exit(_exit_code)
else:
# worker process
# for avoiding multiprocessing.manager process's join error
os._exit(lifetime._exit_code)
from multiprocessing import RLock
import xmlrpc.client as xmlrpclib
from rs4.producers import file_producer
from .api import DateEncoder
if os.environ.get("SKITAI_ENV") == "PYTEST":
from .semaps import TestSemaps as Semaps
else:
from .semaps import Semaps
if os.name == "nt":
TEMP_DIR = os.path.join(tempfile.gettempdir(), "skitai-gentemp")
else:
TEMP_DIR = "/var/tmp/skitai-gentemp"
pathtool.mkdir(TEMP_DIR)
composer.Composer.SAVE_PATH = os.path.join("/var/tmp/skitai", "smtpda",
"spool")
pathtool.mkdir(composer.Composer.SAVE_PATH)
class WASBase:
version = __version__
objects = {}
_luwatcher = Semaps([], "d", 256)
lock = plock = RLock()
init_time = time.time()
cloned = False
_process_locks = [RLock() for i in range(8)]
def email(self, subject, snd, rcpt):
if composer.Composer.SAVE_PATH is None:
composer.Composer.SAVE_PATH = os.path.join("/var/tmp/skitai",
"smtpda", "spool")
pathtool.mkdir(composer.Composer.SAVE_PATH)
return composer.Composer(subject, snd, rcpt)
from .triple_logger import Logger
from . import django_adaptor
if os.environ.get("SKITAI_ENV") == "PYTEST":
from threading import RLock
else:
from multiprocessing import RLock
import xmlrpc.client as xmlrpclib
from rs4.producers import file_producer
from .api import DateEncoder
if os.name == "nt":
TEMP_DIR = os.path.join(tempfile.gettempdir(), "skitai-gentemp")
else:
TEMP_DIR = "/var/tmp/skitai-gentemp"
pathtool.mkdir(TEMP_DIR)
class WASBase:
version = __version__
objects = {}
_luwatcher = None
_stwatcher = None
lock = plock = RLock()
init_time = time.time()
cloned = False
_process_locks = [RLock() for i in range(8)]
@classmethod