def _open_logs(id_, no_detach): """Open Cylc log handlers for a flow run.""" if not no_detach: while LOG.handlers: LOG.handlers[0].close() LOG.removeHandler(LOG.handlers[0]) log_path = get_workflow_run_log_name(id_) LOG.addHandler(TimestampRotatingFileHandler(log_path, no_detach)) # Add file installation log file_install_log_path = get_workflow_file_install_log_name(id_) RSYNC_LOG.addHandler( TimestampRotatingFileHandler(file_install_log_path, no_detach))
def daemonize(schd): """Turn a cylc scheduler into a Unix daemon. Do the UNIX double-fork magic, see Stevens' "Advanced Programming in the UNIX Environment" for details (ISBN 0201563177) ATTRIBUTION: base on a public domain code recipe by Jurgen Hermann: http://code.activestate.com/recipes/66012-fork-a-daemon-process-on-unix/ """ logfname = get_workflow_run_log_name(schd.workflow) try: old_log_mtime = os.stat(logfname).st_mtime except OSError: old_log_mtime = None # fork 1 try: pid = os.fork() if pid > 0: # Poll for workflow log to be populated workflow_pid = None workflow_url = None pub_url = None timeout = time() + _TIMEOUT while time() <= timeout and (workflow_pid is None or workflow_url is None or pub_url is None): sleep(0.1) try: # First INFO line of workflow log should contain # start up message, URL and PID. Format is: # LOG-PREFIX Workflow schd program: url=URL, pid=PID # Otherwise, something is wrong, print the workflow log # and exit with an error. log_stat = os.stat(logfname) if (log_stat.st_mtime == old_log_mtime or log_stat.st_size == 0): continue for line in open(logfname): if schd.START_MESSAGE_PREFIX in line: workflow_url, workflow_pid = (item.rsplit( "=", 1)[-1] for item in line.rsplit()[-2:]) if schd.START_PUB_MESSAGE_PREFIX in line: pub_url = line.rsplit("=", 1)[-1].rstrip() if workflow_url and pub_url: break elif ' ERROR -' in line or ' CRITICAL -' in line: # ERROR and CRITICAL before workflow starts try: sys.stderr.write(open(logfname).read()) sys.exit(1) except IOError: sys.exit("Workflow schd program exited") except (IOError, OSError, ValueError): pass if workflow_pid is None or workflow_url is None: sys.exit("Workflow not started after %ds" % _TIMEOUT) # Print workflow information info = { "workflow": schd.workflow, "host": schd.host, "url": workflow_url, "pub_url": pub_url, "ps_opts": PS_OPTS, "pid": workflow_pid } if schd.options.format == 'json': sys.stdout.write(json.dumps(info, indent=4)) else: sys.stdout.write(WORKFLOW_INFO_TMPL % info) # exit parent 1 sys.exit(0) except OSError as exc: sys.exit("fork #1 failed: %d (%s)\n" % (exc.errno, exc.strerror)) # decouple from parent environment os.chdir("/") os.setsid() os.umask(0) # fork 2 try: pid = os.fork() if pid > 0: # exit parent 2 sys.exit(0) except OSError as exc: sys.exit("fork #2 failed: %d (%s)\n" % (exc.errno, exc.strerror)) # reset umask, octal os.umask(0o22) # Redirect /dev/null to stdin. # Note that simply reassigning the sys streams is not sufficient # if we import modules that write to stdin and stdout from C # code - evidently the subprocess module is in this category! # TODO: close resource? atexit? dvnl = open(os.devnull, 'r') os.dup2(dvnl.fileno(), sys.stdin.fileno())
def main(parser: COP, options: 'Values', reg: str, task_id: Optional[str] = None, color: bool = False) -> None: """Implement cylc cat-log CLI. Determine log path, user@host, batchview_cmd, and action (print, dir-list, cat, edit, or tail), and then if the log path is: a) local: perform action on log path, or b) remote: re-invoke cylc cat-log as a) on the remote account """ if options.remote_args: # Invoked on job hosts for job logs only, as a wrapper to view_log(). # Tail and batchview commands from global config on workflow host). logpath, mode, tail_tmpl = options.remote_args[0:3] logpath = expand_path(logpath) tail_tmpl = expand_path(tail_tmpl) try: batchview_cmd = options.remote_args[3] except IndexError: batchview_cmd = None res = view_log(logpath, mode, tail_tmpl, batchview_cmd, remote=True, color=color) if res == 1: sys.exit(res) return workflow_name, _ = parse_reg(reg) # Get long-format mode. try: mode = MODES[options.mode] except KeyError: mode = options.mode if not task_id: # Cat workflow logs, local only. if options.filename is not None: raise UserInputError("The '-f' option is for job logs only.") logpath = get_workflow_run_log_name(workflow_name) if options.rotation_num: logs = glob('%s.*' % logpath) logs.sort(key=os.path.getmtime, reverse=True) try: logpath = logs[int(options.rotation_num)] except IndexError: raise UserInputError("max rotation %d" % (len(logs) - 1)) tail_tmpl = os.path.expandvars(get_platform()["tail command template"]) out = view_log(logpath, mode, tail_tmpl, color=color) if out == 1: sys.exit(1) if mode == 'edit': tmpfile_edit(out, options.geditor) return if task_id: # Cat task job logs, may be on workflow or job host. if options.rotation_num is not None: raise UserInputError("only workflow (not job) logs get rotated") try: task, point = TaskID.split(task_id) except ValueError: parser.error("Illegal task ID: %s" % task_id) if options.submit_num != NN: try: options.submit_num = "%02d" % int(options.submit_num) except ValueError: parser.error("Illegal submit number: %s" % options.submit_num) if options.filename is None: options.filename = JOB_LOG_OUT else: # Convert short filename args to long (e.g. 'o' to 'job.out'). with suppress(KeyError): options.filename = JOB_LOG_OPTS[options.filename] # KeyError: Is already long form (standard log, or custom). platform_name, job_runner_name, live_job_id = get_task_job_attrs( workflow_name, point, task, options.submit_num) platform = get_platform(platform_name) batchview_cmd = None if live_job_id is not None: # Job is currently running. Get special job runner log view # command (e.g. qcat) if one exists, and the log is out or err. conf_key = None if options.filename == JOB_LOG_OUT: if mode == 'cat': conf_key = "out viewer" elif mode == 'tail': conf_key = "out tailer" elif options.filename == JOB_LOG_ERR: if mode == 'cat': conf_key = "err viewer" elif mode == 'tail': conf_key = "err tailer" if conf_key is not None: batchview_cmd_tmpl = None with suppress(KeyError): batchview_cmd_tmpl = platform[conf_key] if batchview_cmd_tmpl is not None: batchview_cmd = batchview_cmd_tmpl % { "job_id": str(live_job_id) } log_is_remote = (is_remote_platform(platform) and (options.filename != JOB_LOG_ACTIVITY)) log_is_retrieved = (platform['retrieve job logs'] and live_job_id is None) if log_is_remote and (not log_is_retrieved or options.force_remote): logpath = os.path.normpath( get_remote_workflow_run_job_dir(workflow_name, point, task, options.submit_num, options.filename)) tail_tmpl = platform["tail command template"] # Reinvoke the cat-log command on the remote account. cmd = ['cat-log', *verbosity_to_opts(cylc.flow.flags.verbosity)] for item in [logpath, mode, tail_tmpl]: cmd.append('--remote-arg=%s' % shlex.quote(item)) if batchview_cmd: cmd.append('--remote-arg=%s' % shlex.quote(batchview_cmd)) cmd.append(workflow_name) is_edit_mode = (mode == 'edit') # TODO: Add Intelligent Host selection to this try: proc = remote_cylc_cmd(cmd, platform, capture_process=is_edit_mode, manage=(mode == 'tail')) except KeyboardInterrupt: # Ctrl-C while tailing. pass else: if is_edit_mode: # Write remote stdout to a temp file for viewing in editor. # Only BUFSIZE bytes at a time in case huge stdout volume. out = NamedTemporaryFile() data = proc.stdout.read(BUFSIZE) while data: out.write(data) data = proc.stdout.read(BUFSIZE) os.chmod(out.name, S_IRUSR) out.seek(0, 0) else: # Local task job or local job log. logpath = os.path.normpath( get_workflow_run_job_dir(workflow_name, point, task, options.submit_num, options.filename)) tail_tmpl = os.path.expandvars(platform["tail command template"]) out = view_log(logpath, mode, tail_tmpl, batchview_cmd, color=color) if mode != 'edit': sys.exit(out) if mode == 'edit': tmpfile_edit(out, options.geditor)