def get_revision():
"""
GET THE CURRENT GIT REVISION
"""
proc = Process("git log", ["git", "log", "-1"])
try:
while True:
line = proc.stdout.pop().strip().decode('utf8')
if not line:
continue
if line.startswith("commit "):
return line[7:]
finally:
with suppress_exception:
proc.join()
def get_branch():
"""
GET THE CURRENT GIT BRANCH
"""
proc = Process("git status", ["git", "status"])
try:
while True:
raw_line = proc.stdout.pop()
line = raw_line.decode('utf8').strip()
if line.startswith("On branch "):
return line[10:]
finally:
try:
proc.join()
except Exception:
pass
def get_remote_revision(url, branch):
"""
GET REVISION OF A REMOTE BRANCH
"""
proc = Process("git remote revision", ["git", "ls-remote", url, "refs/heads/" + branch])
try:
while True:
raw_line = proc.stdout.pop()
line = raw_line.strip().decode('utf8')
if not line:
continue
return line.split("\t")[0]
finally:
try:
proc.join()
except Exception:
pass
def app():
global app_process
pythonpath = str("." + os.pathsep + "vendor")
if PY2:
app_process = Null
elif not app_process:
app_process = Process(
"TUID app",
["python", "tuid/app.py"],
env={str("PYTHONPATH"): pythonpath},
debug=True
)
for line in app_process.stderr:
if line.startswith(b' * Running on '):
break
yield
app_process.stop()
app_process.join(raise_on_error=False)
def __init__(self, name, config):
config = wrap(config)
if config.debug.logs:
Log.error("not allowed to configure logging on other process")
self.process = Process(name, [PYTHON, "mo_threads" + os.sep + "python_worker.py"], shell=True)
self.process.stdin.add(value2json(set_default({"debug": {"trace": True}}, config)))
self.lock = Lock("wait for response from "+name)
self.current_task = None
self.current_response = None
self.current_error = None
self.daemon = Thread.run("", self._daemon)
self.errors = Thread.run("", self._stderr)
class Python(object):
def __init__(self, name, config):
config = wrap(config)
if config.debug.logs:
Log.error("not allowed to configure logging on other process")
self.process = Process(name, [PYTHON, "mo_threads" + os.sep + "python_worker.py"], shell=True)
self.process.stdin.add(value2json(set_default({"debug": {"trace": True}}, config)))
self.lock = Lock("wait for response from "+name)
self.current_task = None
self.current_response = None
self.current_error = None
self.daemon = Thread.run("", self._daemon)
self.errors = Thread.run("", self._stderr)
def _execute(self, command):
with self.lock:
if self.current_task is not None:
self.current_task.wait()
self.current_task = Signal()
self.current_response = None
self.current_error = None
self.process.stdin.add(value2json(command))
self.current_task.wait()
with self.lock:
try:
if self.current_error:
Log.error("problem with process call", cause=Except.new_instance(self.current_error))
else:
return self.current_response
finally:
self.current_task = None
self.current_response = None
self.current_error = None
def _daemon(self, please_stop):
while not please_stop:
line = self.process.stdout.pop(till=please_stop)
if line == THREAD_STOP:
break
try:
data = json2value(line.decode('utf8'))
if "log" in data:
Log.main_log.write(*data.log)
elif "out" in data:
with self.lock:
self.current_response = data.out
self.current_task.go()
elif "err" in data:
with self.lock:
self.current_error = data.err
self.current_task.go()
except Exception:
Log.note("non-json line: {{line}}", line=line)
DEBUG and Log.note("stdout reader is done")
def _stderr(self, please_stop):
while not please_stop:
try:
line = self.process.stderr.pop(till=please_stop)
if line == THREAD_STOP:
please_stop.go()
break
Log.note("Error line from {{name}}({{pid}}): {{line}}", line=line, name=self.process.name, pid=self.process.pid)
except Exception as e:
Log.error("could not process line", cause=e)
def import_module(self, module_name, var_names=None):
if var_names is None:
self._execute({"import": module_name})
else:
self._execute({"import": {"from": module_name, "vars": var_names}})
def set(self, var_name, value):
self._execute({"set": {var_name, value}})
def get(self, var_name):
return self._execute({"get": var_name})
def execute_script(self, script):
return self._execute({"exec": script})
def __getattr__(self, item):
def output(*args, **kwargs):
if len(args):
if len(kwargs.keys()):
Log.error("Not allowed to use both args and kwargs")
return self._execute({item: args})
else:
return self._execute({item: kwargs})
return output
def stop(self):
self._execute({"stop": {}})
self.process.join()
self.daemon.stop()
self.errors.stop()
def local(self, cmd, args, raise_on_error=True):
p = Process(cmd, args,
cwd=self.directory).join(raise_on_error=raise_on_error)
return p, list(p.stdout), list(p.stderr)
class Schedule(object):
@override
def __init__(
self,
interval, # TIME INTERVAL BETWEEN RUNS
starting, # THE TIME TO START THE INTERVAL COUNT
max_runtime=MAX_RUNTIME, # LIMIT HOW LONG THE PROCESS IS ALIVE
wait_for_shutdown=WAIT_FOR_SHUTDOWN, # LIMIT PAITENCE WHEN ASKING FOR SHUTDOWN, THEN SEND KILL
process=None, # PARAMETERS TO START PROCESS
):
self.duration = Duration(interval)
self.starting = coalesce(Date(starting), Date.now())
self.max_runtime = Duration(max_runtime)
self.wait_for_shutdown = Duration(wait_for_shutdown)
# Process parameters
self.process = process
# STATE
self.last_started = None
self.last_finished = None
self.run_count = 0
self.fail_count = 0
self.current = None
self.terminator = None # SIGNAL TO KILL THE PROCESS
self.next_run = self._next_run()
self.next = Till(till=self.next_run)
self.next_run.then(self.run)
def _next_run_time(self):
"""
:return: return signal for next
"""
interval = mo_math.floor((Date.now() - self.starting) / self.duration)
next_time = self.starting + (interval * self.duration)
return next_time
def run(self):
self.last_started = Date.now()
self.run_count += 1
self.current = Process(**self.process)
self.terminator = Till(seconds=self.max_runtime.seconds)
self.terminator.then(self.killer)
self.current.service_stopped.then(self.done)
def killer(self, please_stop):
self.current.stop()
(please_stop
| self.current.service_stopped()
| Till(seconds=self.wait_for_shutdown.seconds)).wait()
if not self.current.service_stopped:
self.fail_count += 1
self.current.kill()
self.current.join()
def done(self):
self.last_finished = Date.now()
self.terminator.remove_go(self.killer)
self.terminator = None
self.current = None
self.next_run = self._next_run_time()
self.next = Till(till=self.next_run.unix)
self.next.then(self.run)
def status(self):
if self.current is None:
status = "never started"
elif not self.current.service_stopped:
status = "running"
elif self.current.returncode == 0:
status = "done"
else:
status = "failed " + text(self.current.returncode)
return Data(
name=self.name,
next_run=self.next_run,
last_started=self.last_started,
last_finished=self.last_finished,
status=status,
)
class Python(object):
def __init__(self, name, config):
config = wrap(config)
if config.debug.logs:
Log.error("not allowed to configure logging on other process")
Log.note("begin process")
# WINDOWS REQUIRED shell, WHILE LINUX NOT
shell = "windows" in platform.system().lower()
self.process = Process(
name, [PYTHON, "-u", "mo_threads" + os.sep + "python_worker.py"],
debug=False,
cwd=os.getcwd(),
shell=shell)
self.process.stdin.add(
value2json(set_default({}, config, {"debug": {
"trace": True
}})))
status = self.process.stdout.pop()
if status != '{"out":"ok"}':
Log.error("could not start python\n{{error|indent}}",
error=self.process.stderr.pop_all() + [status] +
self.process.stdin.pop_all())
self.lock = Lock("wait for response from " + name)
self.current_task = DONE
self.current_response = None
self.current_error = None
self.daemon = Thread.run("", self._daemon)
self.errors = Thread.run("", self._stderr)
def _execute(self, command):
with self.lock:
self.current_task.wait()
self.current_task = Signal()
self.current_response = None
self.current_error = None
if self.process.service_stopped:
Log.error("python is not running")
self.process.stdin.add(value2json(command))
(self.current_task | self.process.service_stopped).wait()
try:
if self.current_error:
Log.error("problem with process call",
cause=Except.new_instance(self.current_error))
else:
return self.current_response
finally:
self.current_task = DONE
self.current_response = None
self.current_error = None
def _daemon(self, please_stop):
while not please_stop:
line = self.process.stdout.pop(till=please_stop)
if line == THREAD_STOP:
break
try:
data = json2value(line)
if "log" in data:
Log.main_log.write(*data.log)
elif "out" in data:
self.current_response = data.out
self.current_task.go()
elif "err" in data:
self.current_error = data.err
self.current_task.go()
except Exception as e:
Log.note("non-json line: {{line}}", line=line)
DEBUG and Log.note("stdout reader is done")
def _stderr(self, please_stop):
while not please_stop:
try:
line = self.process.stderr.pop(till=please_stop)
if line == THREAD_STOP:
please_stop.go()
break
Log.note("Error line from {{name}}({{pid}}): {{line}}",
line=line,
name=self.process.name,
pid=self.process.pid)
except Exception as e:
Log.error("could not process line", cause=e)
def import_module(self, module_name, var_names=None):
if var_names is None:
self._execute({"import": module_name})
else:
self._execute({"import": {"from": module_name, "vars": var_names}})
def set(self, var_name, value):
self._execute({"set": {var_name, value}})
def get(self, var_name):
return self._execute({"get": var_name})
def execute_script(self, script):
return self._execute({"exec": script})
def __getattr__(self, item):
def output(*args, **kwargs):
if len(args):
if kwargs.keys():
Log.error("Not allowed to use both args and kwargs")
return self._execute({item: args})
else:
return self._execute({item: kwargs})
return output
def stop(self):
self._execute({"stop": {}})
self.process.join()
self.daemon.stop()
self.errors.stop()