def __init__(self):
cf = load_program_configuration()
self.queuedir = load_cf_param(cf, 'queuedir', lambda f: os.path.isdir(f))
INotifyWatch.__init__(self, self.queuedir, IN_CREATE)
(obj, func, filename, line) = callerinfo()
self.plugin_manager = PluginManager(os.path.join(os.path.abspath(os.path.dirname(filename)), 'queue_plugins'))
debug("initialized")
def __init__(self, plugin_dir, watchfolder=False):
if not os.path.exists(plugin_dir) or not os.path.isdir(plugin_dir):
raise RuntimeError("can't read plugin_dir %s" % plugin_dir)
self.plugin_dir = plugin_dir
self.scandir(self.plugin_dir)
debug("initializing plugins from folder %s" % plugin_dir)
if watchfolder:
self.watch_plugins()
def callback(self, events):
debug("spawner: callback(pid=%d,events=%d,fd=%d)" % (self.pid, events, self.fd))
if events & EVREAD:
debug("spawner: EVREAD")
# TODO: reading from fh=fdopen(self.fd) was causing too much of a buffering, check WHY
buf = os.read(self.fd, 32768)
print("CHILD-%d: %s" % (self.pid, buf))
more = events & (~EVREAD)
if more:
debug("CHILD %d died" % self.pid)
self.close()
try:
(epid, status) = os.waitpid(self.pid, 0)
except OSError as ose:
debug(ose)
pass
else:
sig = status & 0x0f
rc = (status >> 8) & 0x0f
debug("pid=%d, status=%d, sig=%d, rc=%d" % (epid, status, sig, rc))
def load_program_configuration(envname=None):
"""
:param envname: str
:return: object
envname can contain configuration's module name or path to the configuration name
if envname is not specified, a default TASTYPY_<PROGRAM_NAME>_CF env var is queried
if no env is found then program name is used as the name of configuration file
"""
# TODO: isolate configuration loading in its own namespace for security reason (clean namespace pollution)
#
if envname is None:
envname = '_'.join(['TASTYPY', tastypy_program_name().upper(), 'CF'])
debug("envname=%s" % envname)
cfname = os.getenv(envname, tastypy_program_name().lower())
debug("configmodule=%s" % cfname)
if os.path.exists(cfname):
if os.path.isdir(cfname):
sys.path.append(cfname)
cfname = sys.argv[0].lower()
elif os.path.isfile(cfname):
sys.path.append(os.path.dirname(cfname))
(cfname, ext) = os.path.basename(cfname).split('.')
if not ext.startswith('py'):
raise RuntimeError("config seems to be a filepath but does not end with .py*")
else:
raise RuntimeError("config is a path but not a folder or regular file")
cf = None
try:
cf = __import__(cfname)
except ImportError:
raise RuntimeError("error importing configuration file")
debug("configuration loaded at %r" % cf)
return cf
def child_event(self, fd, events):
"""
called when a child writes to stdout or exits
:param fd: int
:param events: int
:return: None
"""
pid = self.pipes[fd]
child = self.runners[pid]
debug("spawner: callback(pid=%d,events=%d,fd=%d)" % (pid, events, fd))
if events & EVREAD:
debug("spawner: EVREAD")
# TODO: reading from fh=fdopen(self.fd) was causing too much of a buffering, check WHY
buf = os.read(fd, 32768)
print("CHILD-%d: %s" % (pid, buf))
self.runners[pid]['taskcount'] += 1
if self.runners[pid]['taskcount'] == 3:
self.runners[pid]['state'] = 'RESTARTING'
os.kill(pid, SIGTERM)
more = events & (~EVREAD)
if more:
debug("CHILD %d died" % pid)
self.close(fd)
try:
(epid, status) = os.waitpid(pid, 0)
except OSError as ose:
raise ose
# debug(ose)
# pass
else:
sig = status & 0x0f
rc = (status >> 8) & 0x0f
debug("pid=%d, status=%d, sig=%d, rc=%d" % (epid, status, sig, rc))
self.pipes.pop(fd, None)
self.runners.pop(pid, None)
if child['state'] == 'RESTARTING':
self.spawn()
def callback(self, wd, mask, cookie, filename):
debug("NEWTASK: %s" % filename)
