def run(self, meta): if self.interractive: os.system(self.cmd) else: child = Process(self.cmd) events = multiprocessing.Queue() parent = multiprocessing.Process(name="task", target=child.run, args=(events, True)) parent.start() exit = False returncode = None while not exit: try: msg = events.get(timeout=0.1) except Empty: if exit: break else: if msg.type == "line": print(term.lightblack("\u2502"), msg.data.decode("utf-8"), end="") elif msg.type == "start": print("$ " + term.lightwhite(self.cmd) + term.black(" # pid=%s" % msg.data["pid"])) elif msg.type == "stop": returncode = msg.data["returncode"] if returncode: print(term.lightblack("\u2514" + term.red(" failed (rc={}). ".format(returncode)))) else: print(term.lightblack("\u2514" + term.green(" success. "))) exit = True if returncode: raise RuntimeError( '"{command}" exited with status {returncode}.'.format(command=self.cmd, returncode=returncode) ) self.set_complete()
def __init__(self): tokens = ( '%(color)s%(levelname)s', '%(spent)04d', '%(name)s', ) sep = '|' if iswindows else (Style.RESET_ALL + lightblack(':')) self._fmt = sep.join(tokens) + lightblack(':') + ' %(message)s' + EOL.strip()
def handle(self, *args, **options): _stdout_backup, _stderr_backup = self.stdout, self.stderr self.stdout = OutputWrapper(ConsoleOutputPlugin._stdout, ending=CLEAR_EOL + '\n') self.stderr = OutputWrapper(ConsoleOutputPlugin._stderr, ending=CLEAR_EOL + '\n') self.stderr.style_func = lambda x: Fore.LIGHTRED_EX + Back.RED + '!' + Style.RESET_ALL + ' ' + x with bonobo.parse_args(options) as options: services = self.get_services() graph_coll = self.get_graph(*args, **options) if not isinstance(graph_coll, GeneratorType): graph_coll = (graph_coll, ) for i, graph in enumerate(graph_coll): assert isinstance(graph, bonobo.Graph), 'Invalid graph provided.' print(term.lightwhite('{}. {}'.format(i + 1, graph.name))) result = bonobo.run(graph, services=services) print(term.lightblack(' ... return value: ' + str(result))) print() self.stdout, self.stderr = _stdout_backup, _stderr_backup
def run(self, *args, **options): results = [] with bonobo.parse_args(options) as options: services = self.get_services() strategy = self.get_strategy() graph_coll = self.get_graph(*args, **options) if not isinstance(graph_coll, GeneratorType): graph_coll = (graph_coll, ) for i, graph in enumerate(graph_coll): if not isinstance(graph, bonobo.Graph): raise ValueError( "Expected a Graph instance, got {!r}.".format(graph)) print( term.lightwhite("{}. {}".format( i + 1, graph.name or repr(graph).strip("<>")))) result = bonobo.run(graph, services=services, strategy=strategy) results.append(result) for node in result.nodes: print(node.get_statistics_as_string(), node.get_flags_as_string()) print(term.lightblack(" ... return value: " + str(result))) return results
def format_arg(arg): length = len(pre_re.sub("\\1\\2\\3", arg)) arg = pre_re.sub(w("\\1") + term.bold("\\2") + w("\\3"), arg) arg = re.sub(r"^ \$ (.*)", term.lightblack(" $ ") + term.reset("\\1"), arg) return (arg, length)
def get_flags_as_string(self): if self._defunct: return term.red('[defunct]') if self.killed: return term.lightred('[killed]') if self.stopped: return term.lightblack('[done]') return ''
def get_flags_as_string(self): if self._defunct: return term.red("[defunct]") if self.killed: return term.lightred("[killed]") if self.stopped: return term.lightblack("[done]") return ""
def run(self, meta): if self.interractive: os.system(self.cmd) else: child = Process(self.cmd) events = multiprocessing.Queue() parent = multiprocessing.Process(name='task', target=child.run, args=(events, True)) parent.start() exit = False returncode = None while not exit: try: msg = events.get(timeout=0.1) except Empty: if exit: break else: if msg.type == 'line': print(term.lightblack('\u2502'), msg.data.decode('utf-8'), end='') elif msg.type == 'start': print('$ ' + term.lightwhite(self.cmd) + term.black(' # pid=%s' % msg.data['pid'])) elif msg.type == 'stop': returncode = msg.data['returncode'] if returncode: print( term.lightblack('\u2514' + term.red( ' failed (rc={}). '.format(returncode)))) else: print( term.lightblack('\u2514' + term.green(' success. '))) exit = True if returncode: raise RuntimeError( '"{command}" exited with status {returncode}.'.format( command=self.cmd, returncode=returncode)) self.set_complete()
def formatException(self, excinfo): formatted_exception = format_exception(*excinfo) output = [] stack_length = len(formatted_exception) for i, frame in enumerate(formatted_exception): if frame.startswith(' '): output.append(textwrap.indent(' ' + frame.strip(), lightblack('\u2502 '))) else: g = re.match('([a-zA-Z.]+): (.*)$', frame.strip(), flags=re.DOTALL) if g is not None: etyp, emsg = g.group(1), g.group(2) output.append( lightblack('\u2514' if i + 1 == stack_length else '\u251c') + lightblack_bg(lightwhite(' ' + etyp + ' ')) + ' ' + lightwhite(textwrap.indent(str(emsg), ' ' * (len(etyp) + 4)).strip()) ) else: output.append(textwrap.indent(frame.strip(), lightblack('\u2502 '))) return EOL.join(output)
def execute(self, command, *, cwd, shell=True): self.logger.info( term.bold(">>> %s") + " " + term.lightblack("(in %s)"), command, cwd) retval = subprocess.call(command, cwd=cwd, shell=shell) if retval: self.logger.error(term.red(term.bold("... ✖ failed"))) raise RuntimeError('"{}" returned {}.'.format(command, retval)) else: self.logger.info(term.green(term.bold("... ✓ ok")))
def dispatch(self, event_id, event=None): should_log = not event_id.startswith("medikit.on_file_") or self.logger.getEffectiveLevel() <= logging.DEBUG if should_log: self.logger.info( self.indent + term.bold(">") + " dispatch ⚡ {} ({})".format(term.bold(term.blue(event_id)), type(event or Event).__name__) ) type(self).indent_level += 1 event = super(LoggingDispatcher, self).dispatch(event_id, event) type(self).indent_level -= 1 if should_log: self.logger.info(self.indent + term.bold("<") + " {}".format(term.lightblack("dispatched " + event_id))) return event
def _handle_continue(cls, pipeline, *, filename): if not os.path.exists(filename): raise FileNotFoundError( 'Pipeline “{}” not started, hence you cannot “continue” it. Are you looking for `medikit pipeline {name} start`?'. format(name=pipeline.name) ) # XXX TODO add a lock file during the step and unlock at the end. with open(filename) as f: pipeline.unserialize(f.read()) try: step = pipeline.next() name, current, size, descr = pipeline.name, pipeline.current, len(pipeline), str(step) logger.info( term.black(' » ').join( ( term.lightblue('{} ({}/{})'.format(name.upper(), current, size)), term.yellow('⇩ BEGIN'), term.lightblack(descr), ) ) ) step.run(pipeline.meta) if step.complete: logger.info( term.black(' » ') .join(( term.lightblue('{} ({}/{})'.format(name.upper(), current, size)), term.green('SUCCESS'), )) + '\n' ) else: logger.info( term.black(' » ') .join(( term.lightblue('{} ({}/{})'.format(name.upper(), current, size)), term.red('FAILED'), )) + '\n' ) return except StopIteration: return COMPLETE with open(filename, 'w+') as f: f.write(pipeline.serialize()) return CONTINUE
def dispatch(self, event_id, event=None): should_log = not event_id.startswith('medikit.on_file_') or \ self.logger.getEffectiveLevel() <= logging.DEBUG if should_log: self.logger.info( self.indent + term.bold('>') + ' dispatch ⚡ {} ({})'.format(term.bold(term.blue(event_id)), type(event or Event).__name__) ) type(self).indent_level += 1 event = super(LoggingDispatcher, self).dispatch(event_id, event) type(self).indent_level -= 1 if should_log: self.logger.info(self.indent + term.bold('<') + ' {}'.format(term.lightblack('dispatched ' + event_id))) return event
def _handle_continue(cls, pipeline, *, filename): if not os.path.exists(filename): raise FileNotFoundError( "Pipeline “{}” not started, hence you cannot “continue” it. Are you looking for `medikit pipeline {name} start`?".format( name=pipeline.name ) ) # XXX TODO add a lock file during the step and unlock at the end. with open(filename) as f: pipeline.unserialize(f.read()) try: step = pipeline.next() name, current, size, descr = pipeline.name, pipeline.current, len(pipeline), str(step) logger.info( term.black(" » ").join( ( term.lightblue("{} ({}/{})".format(name.upper(), current, size)), term.yellow("⇩ BEGIN"), term.lightblack(descr), ) ) ) step.run(pipeline.meta) if step.complete: logger.info( term.black(" » ").join( (term.lightblue("{} ({}/{})".format(name.upper(), current, size)), term.green("SUCCESS")) ) + "\n" ) else: logger.info( term.black(" » ").join( (term.lightblue("{} ({}/{})".format(name.upper(), current, size)), term.red("FAILED")) ) + "\n" ) return except StopIteration: return COMPLETE with open(filename, "w+") as f: f.write(pipeline.serialize()) return CONTINUE
def dispatch(self, event_id, event=None): should_log = not event_id.startswith( "medikit.on_file_" ) or self.logger.getEffectiveLevel() <= logging.DEBUG if should_log: self.logger.info( self.indent + term.bold(">") + " dispatch ⚡ {} ({})".format(term.bold(term.blue(event_id)), type(event or Event).__name__)) type(self).indent_level += 1 event = super(LoggingDispatcher, self).dispatch(event_id, event) type(self).indent_level -= 1 if should_log: self.logger.info( self.indent + term.bold("<") + " {}".format(term.lightblack("dispatched " + event_id))) return event
def format_console(self, index, key, value, *, fields=None): fields = fields or [] if not isinstance(key, str): if len(fields) > key and str(key) != str(fields[key]): key = '{}{}'.format(fields[key], term.lightblack('[{}]'.format(key))) else: key = str(index) prefix = '\u2502 {} = '.format(key) prefix_length = len(prefix) def indent(text, prefix): for i, line in enumerate(text.splitlines()): yield (prefix if i else '') + line + CLEAR_EOL + '\n' repr_of_value = ''.join( indent(pprint.pformat(value, width=self.max_width - prefix_length), '\u2502' + ' ' * (len(prefix) - 1)) ).strip() return '{}{}{}'.format(prefix, repr_of_value.replace('\n', CLEAR_EOL + '\n'), CLEAR_EOL)
def run(self, *args, **options): results = [] with bonobo.parse_args(options) as options: services = self.get_services() strategy = self.get_strategy() graph_coll = self.get_graph(*args, **options) if not isinstance(graph_coll, GeneratorType): graph_coll = (graph_coll, ) for i, graph in enumerate(graph_coll): assert isinstance(graph, bonobo.Graph), 'Invalid graph provided.' print(term.lightwhite('{}. {}'.format(i + 1, graph.name))) result = bonobo.run(graph, services=services, strategy=strategy) results.append(result) print(term.lightblack(' ... return value: ' + str(result))) print() return results
def run(self, *args, **options): results = [] with bonobo.parse_args(options) as options: services = self.get_services() strategy = self.get_strategy() graph_coll = self.get_graph(*args, **options) if not isinstance(graph_coll, GeneratorType): graph_coll = (graph_coll,) for i, graph in enumerate(graph_coll): if not isinstance(graph, bonobo.Graph): raise ValueError('Expected a Graph instance, got {!r}.'.format(graph)) print(term.lightwhite('{}. {}'.format(i + 1, graph.name))) result = bonobo.run(graph, services=services, strategy=strategy) results.append(result) print(term.lightblack(' ... return value: ' + str(result))) print() return results
def format_console(self, index, key, value, *, fields=None): fields = fields or [] if not isinstance(key, str): if len(fields) > key and str(key) != str(fields[key]): key = "{}{}".format(fields[key], term.lightblack("[{}]".format(key))) else: key = str(index) prefix = "\u2502 {} = ".format(key) prefix_length = len(prefix) def indent(text, prefix): for i, line in enumerate(text.splitlines()): yield (prefix if i else "") + line + CLEAR_EOL + "\n" repr_of_value = "".join( indent(pprint.pformat(value, width=self.max_width - prefix_length), "\u2502" + " " * (len(prefix) - 1))).strip() return "{}{}{}".format(prefix, repr_of_value.replace("\n", CLEAR_EOL + "\n"), CLEAR_EOL)
def __init__(self): tokens = ("%(color)s%(levelname)s", "%(spent)04d", "%(name)s") sep = "|" if iswindows else (Style.RESET_ALL + lightblack(":")) self._fmt = sep.join(tokens) + lightblack( ":") + " %(message)s" + EOL.strip()
def info(self, *messages): return self.logger.info(self.indent + term.lightblack("∙") + " " + " ".join(map(str, messages)))
def humanized(exc, *, fg=term.red, bg=lambda *args: term.red_bg(term.bold(*args)), help_url=None): SPACES = 2 prefix, suffix = fg(VERT + " " * (SPACES - 1)), fg(" " * (SPACES - 1) + VERT) result = [] def format_arg(arg): length = len(preformatted_pattern.sub("\\1\\2\\3", arg)) arg = preformatted_pattern.sub("\\1" + term.bold("\\2") + "\\3", arg) arg = re.sub("^ \$ (.*)", term.lightblack(" $ ") + term.reset("\\1"), arg) return (arg, length) def joined(*args): return "".join(args) term_width, term_height = term.get_size() line_length = min(80, term_width) for arg in exc.args: line_length = max(min(line_length, len(arg) + 2 * SPACES), 120) result.append(joined(fg(TOP_LEFT + HORIZ * (line_length - 2) + TOP_RIGHT))) args = list(exc.args) for i, arg in enumerate(args): if i == 1: result.append( joined(prefix, " " * (line_length - 2 * SPACES), suffix)) arg_formatted, arg_length = format_arg(arg) if not i: # first line result.append( joined( prefix, bg(" " + type(exc).__name__ + " "), " ", term.white(arg_formatted), " " * (line_length - (arg_length + 3 + len(type(exc).__name__) + 2 * SPACES)), suffix, )) else: # other lines result.append( joined( prefix, arg_formatted + " " * (line_length - arg_length - 2 * SPACES), suffix)) if help_url: help_prefix = "Read more: " arg_length = len(help_url) + len(help_prefix) arg_formatted = help_prefix + term.underline(term.lightblue(help_url)) result.append(joined(prefix, " " * (line_length - 2 * SPACES), suffix)) result.append( joined( prefix, arg_formatted + " " * (line_length - arg_length - 2 * SPACES), suffix)) more = settings.DEBUG exc_lines = format_exception(exc_info(), fg=fg, bg=bg, summary=False).splitlines() if not len(exc_lines): more = False result.append( joined( fg((VERT_LEFT if more else BOTTOM_LEFT) + HORIZ * (line_length - 2) + BOTTOM_RIGHT))) if more: for _line in exc_lines: result.append(_line) result.append(joined(fg("╵"))) elif len(exc_lines): result.append( term.lightblack( "(add DEBUG=1 to system environment for stack trace)".rjust( line_length))) return "\n".join(result)
def __str__(self): return humanized( self, fg=term.green, bg=(lambda *args: term.lightgreen_bg(term.lightblack(*args))), help_url=self.help_url)
def info(self, *messages): return self.logger.info(self.indent + term.lightblack('∙') + ' ' + ' '.join(map(str, messages)))