def identify_main(): usage = None parser = argparse.ArgumentParser(usage=usage) parser.add_argument('--image', default=None) parser.add_argument('--command', default=None) parsed = parser.parse_args() image = parsed.image if image is not None: ni: NodeInfo = identify_image2(image) elif parsed.command is not None: command = parsed.command.split() ni: NodeInfo = identify_command(command) else: msg = 'Please specify either --image or --command' logger.error(msg) sys.exit(1) print('\n\n') print(indent(describe_nd(ni.nd), '', 'desc: ')) print('\n\n') print(indent(describe_bd(ni.bd), '', 'build: ')) print('\n\n') print(indent(describe_cd(ni.cd), '', 'config: ')) print('\n\n') print(indent(describe(ni.pd.data), '', 'data: ')) print('\n\n') print(indent(describe(ni.pd.meta), '', 'meta: '))
def check_compatible_protocol(p1: InteractionProtocol, p2: InteractionProtocol): """ Checks that p1 is a subprotocol of p2 """ try: # check input compatibility # we should have all inputs for k, v2 in p2.inputs.items(): if not k in p1.inputs: msg = f'First protocol misses input "{k}".' raise IncompatibleProtocol(msg) v1 = p1.inputs[k] can, why = can_be_used_as(v1, v2) if not can: msg = f'For input "{k}", cannot use type {v1} as {v2}: {why}' raise IncompatibleProtocol(msg) # check output compatibility # we should have all inputs for k, v2 in p2.outputs.items(): if not k in p1.outputs: msg = f'First protocol misses output "{k}".' raise IncompatibleProtocol(msg) v1 = p1.outputs[k] can, why = can_be_used_as(v1, v2) if not can: msg = f'For output "{k}", cannot use type {v1} as {v2}: {why}' raise IncompatibleProtocol(msg) # XXX: to finish except IncompatibleProtocol as e: msg = 'Cannot say that p1 is a sub-protocol of p2' msg += '\n' + indent(p1, '|', 'p1: |') msg += '\n' + indent(p2, '|', 'p2: |') raise IncompatibleProtocol(msg) from e
def run_example(name, command, expect_fail=False): examples = get_examples_path() pyfile = os.path.join(examples, '%s.py' % name) if not os.path.exists(pyfile): msg = 'Example file does not exist: %s' % pyfile raise Exception(msg) with create_tmp_dir() as cwd: cmd = [pyfile, command] try: res = system_cmd_result(cwd, cmd, display_stdout=False, display_stderr=False, raise_on_error=True) if expect_fail: msg = 'Expected failure of %s but everything OK.' % name msg += '\n cwd = %s' % cwd msg += '\n' + indent(res.stderr, 'stderr| ') msg += '\n' + indent(res.stdout, 'stdout| ') raise Exception(msg) return res except CmdException as e: stderr = e.res.stderr stdout = e.res.stdout if not expect_fail: msg = ('Example %r: Command %r failed unexpectedly.' % (name, command)) msg += '\n retcode: %r' % e.res.ret msg += '\n' + indent(stderr, 'stderr| ') msg += '\n' + indent(stdout, 'stdout| ') raise Exception(msg)
def call_recursive(self, context, child_name, cmd_class, args, extra_dep: List = None, add_outdir=None, add_job_prefix=None, separate_resource_manager=False, separate_report_manager=False, extra_report_keys=None): if extra_dep is None: extra_dep = [] instance = cmd_class() instance.set_parent(self) is_quickapp = isinstance(instance, QuickApp) try: # we are already in a context; just define jobs child_context = context.child( qapp=instance, name=child_name, extra_dep=extra_dep, add_outdir=add_outdir, extra_report_keys=extra_report_keys, separate_resource_manager=separate_resource_manager, separate_report_manager=separate_report_manager, add_job_prefix=add_job_prefix) # XXX if isinstance(args, list): instance.set_options_from_args(args) elif isinstance(args, dict): instance.set_options_from_dict(args) else: assert False if not is_quickapp: self.child_context = child_context res = instance.go() else: instance.context = child_context res = instance.define_jobs_context(child_context) # Add his jobs to our list of jobs context._jobs.update(child_context.all_jobs_dict()) return res except Exception as e: msg = 'While trying to run %s\n' % cmd_class.__name__ msg += 'with arguments = %s\n' % args if '_options' in instance.__dict__: msg += ' parsed options: %s\n' % instance.get_options() msg += ' params: %s\n' % instance.get_options().get_params() if isinstance(e, QuickAppException): msg += indent(str(e), '> ') else: msg += indent(traceback.format_exc(), '> ') raise QuickAppException(msg)
def parmake_job2_new_process(args): """ Starts the job in a new compmake process. """ (job_id, context) = args compmake_bin = which("compmake") db = context.get_compmake_db() storage = db.basepath # XXX: where = os.path.join(storage, "parmake_job2_new_process") if not os.path.exists(storage): try: os.makedirs(storage) except: pass out_result = os.path.join(where, "%s.results.pickle" % job_id) out_result = os.path.abspath(out_result) cmd = [compmake_bin, storage] if not all_disabled(): cmd += ["--contracts"] cmd += [ "--status_line_enabled", "0", "--colorize", "0", "-c", "make_single out_result=%s %s" % (out_result, job_id), ] cwd = os.getcwd() cmd_res = system_cmd_result( cwd, cmd, display_stdout=False, display_stderr=False, raise_on_error=False, capture_keyboard_interrupt=False ) ret = cmd_res.ret if ret == CompmakeConstants.RET_CODE_JOB_FAILED: # XXX: msg = "Job %r failed in external process" % job_id msg += indent(cmd_res.stdout, "stdout| ") msg += indent(cmd_res.stderr, "stderr| ") res = safe_pickle_load(out_result) os.unlink(out_result) result_dict_check(res) raise JobFailed.from_dict(res) elif ret != 0: msg = "Host failed while doing %r" % job_id msg += "\n cmd: %s" % " ".join(cmd) msg += "\n" + indent(cmd_res.stdout, "stdout| ") msg += "\n" + indent(cmd_res.stderr, "stderr| ") raise CompmakeBug(msg) # XXX: res = safe_pickle_load(out_result) os.unlink(out_result) result_dict_check(res) return res
def get(self, timeout=0): # @UnusedVariable if not self.told_you_ready: raise CompmakeBug("I didnt tell you it was ready.") if self.already_read: msg = 'Compmake BUG: should not call twice.' raise CompmakeBug(msg) self.already_read = True assert os.path.exists(self.retcode) ret_str = open(self.retcode, 'r').read() try: ret = int(ret_str) except ValueError: msg = 'Could not interpret file %r: %r.' % (self.retcode, ret_str) raise HostFailed(host='localhost', job_id=self.job_id, reason=msg, bt='') # # # raise HostFailed(host="xxx", # job_id=self.job_id, # reason=reason, bt="") # XXX # try: stderr = open(self.stderr, 'r').read() stdout = open(self.stdout, 'r').read() stderr = 'Contents of %s:\n' % self.stderr + stderr stdout = 'Contents of %s:\n' % self.stdout + stdout # if ret == CompmakeConstants.RET_CODE_JOB_FAILED: # msg = 'SGE Job failed (ret: %s)\n' % ret # msg += indent(stderr, '| ') # # mark_as_failed(self.job_id, msg, None) # raise JobFailed(msg) # elif ret != 0: # msg = 'SGE Job failed (ret: %s)\n' % ret # error(msg) # msg += indent(stderr, '| ') # raise JobFailed(msg) if not os.path.exists(self.out_results): msg = 'job succeeded but no %r found' % self.out_results msg += '\n' + indent(stderr, 'stderr') msg += '\n' + indent(stdout, 'stdout') raise CompmakeBug(msg) res = safe_pickle_load(self.out_results) result_dict_raise_if_error(res) return res finally: fs = [self.stderr, self.stdout, self.out_results, self.retcode] for filename in fs: if os.path.exists(filename): os.unlink(filename)
def get_object_tree( po: PlacedObject, levels: int = 100, spatial_relations: bool = False, attributes: bool = False, ) -> str: ss = [] ss.append("%s" % type(po).__name__) d = po.params_to_json_dict() d.pop("children", None) d.pop("spatial_relations", None) if attributes: if d: ds = yaml.safe_dump( d, encoding="utf-8", indent=4, allow_unicode=True, default_flow_style=False, ) if isinstance(ds, bytes): ds = ds.decode("utf-8") ss.append("\n" + indent(ds, " ")) if po.children and levels >= 1: ss.append("") N = len(po.children) for i, (child_name, child) in enumerate(po.children.items()): if i != N - 1: prefix1 = u"├ %s ┐ " % child_name prefix2 = u"│ %s │ " % (" " * len(child_name)) else: prefix1 = u"└ %s ┐ " % child_name prefix2 = u" %s │ " % (" " * len(child_name)) c = get_object_tree( child, attributes=attributes, spatial_relations=spatial_relations, levels=levels - 1, ) sc = indent(c, prefix2, prefix1) n = max(len(_) for _ in sc.split("\n")) sc += "\n" + prefix2[:-2] + u"└" + u"─" * (n - len(prefix2) + 3) ss.append(sc) if spatial_relations: if po.spatial_relations and levels >= 1: ss.append("") for r_name, rel in po.spatial_relations.items(): ss.append( '- from "%s" to "%s" %s ' % (url_from_fqn(rel.a), url_from_fqn(rel.b), rel.transform)) return "\n".join(ss)
def format_example(e): print('Filename: %s' % e.filename) text = '' # text += '<h2>Example <tt>%s</tt></h2>\n\n' % e.shortname text += e.description text += '\n\n' # text += "MCDPL Code:" code = e.mcdp # code = code.replace('mcdp', '<strong>mcdp</strong>') text += '\n\n' try: text += ast_to_html(code) except BaseException as err: text += '<pre>%s</pre>' % err if False: text += '<pre class="mcdp"><code class="mcdp">\n' def escape(x): x = x.replace('<', '<') return x text += indent(higlight_chunk(escape(code)), ' ' * 4) text += '\n</code></pre>\n' text += '\n\n' # plots = ['ndp_clean'] plots = ['ndp_default'] try: outputs = do_plots(e.filename, plots, outdir, extra_params="") except DPSemanticError as e: text += '\n\n<code class="DPSemanticError">%s</code>\n\n' % indent( str(e), ' ' * 4) except DPInternalError as e: text += '\n\n<code class="DPInternalError">%s</code>\n\n' % indent( str(e), ' ' * 4) else: for ext, name, data in outputs: out = os.path.splitext(e.filename)[0] + '-%s.%s' % (name, ext) with open(out, 'w') as f: f.write(data) if ext in ['png']: imgurl = os.path.relpath(out, os.path.dirname(e.filename)) print('url: %r' % imgurl) text += '\n <img class="output" src="%s"/> \n\n' % (imgurl) text += '\n\n' return text
def call_recursive(self, context, child_name, cmd_class, args, extra_dep=[], add_outdir=None, add_job_prefix=None, separate_resource_manager=False, separate_report_manager=False, extra_report_keys=None): instance = cmd_class() instance.set_parent(self) is_quickapp = isinstance(instance, QuickApp) try: # we are already in a context; just define jobs child_context = context.child(qapp=instance, name=child_name, extra_dep=extra_dep, add_outdir=add_outdir, extra_report_keys=extra_report_keys, separate_resource_manager=separate_resource_manager, separate_report_manager=separate_report_manager, add_job_prefix=add_job_prefix) # XXX if isinstance(args, list): instance.set_options_from_args(args) elif isinstance(args, dict): instance.set_options_from_dict(args) else: assert False if not is_quickapp: self.child_context = child_context res = instance.go() else: instance.context = child_context res = instance.define_jobs_context(child_context) # Add his jobs to our list of jobs context._jobs.update(child_context.all_jobs_dict()) return res except Exception as e: msg = 'While trying to run %s\n' % cmd_class.__name__ msg += 'with arguments = %s\n' % args if '_options' in instance.__dict__: msg += ' parsed options: %s\n' % instance.get_options() msg += ' params: %s\n' % instance.get_options().get_params() if isinstance(e, QuickAppException): msg += indent(str(e), '> ') else: msg += indent(traceback.format_exc(e), '> ') raise QuickAppException(msg)
def format_example(e): print('Filename: %s' % e.filename) text = '' # text += '<h2>Example <tt>%s</tt></h2>\n\n' % e.shortname text += e.description text += '\n\n' # text += "MCDPL Code:" code = e.mcdp # code = code.replace('mcdp', '<strong>mcdp</strong>') text += '\n\n' try: text += ast_to_html(code, complete_document=False) except BaseException as err: text += '<pre>%s</pre>' % err if False: text += '<pre class="mcdp"><code class="mcdp">\n' def escape(x): x = x.replace('<', '<') return x text += indent(higlight_chunk(escape(code)), ' '*4) text += '\n</code></pre>\n' text += '\n\n' # plots = ['ndp_clean'] plots = ['ndp_default'] try: outputs = do_plots(e.filename, plots, outdir, extra_params="") except DPSemanticError as e: text += '\n\n<code class="DPSemanticError">%s</code>\n\n' % indent(str(e), ' ' * 4) except DPInternalError as e: text += '\n\n<code class="DPInternalError">%s</code>\n\n' % indent(str(e), ' ' * 4) else: for ext, name, data in outputs: out = os.path.splitext(e.filename)[0] +'-%s.%s' % (name, ext) with open(out, 'w') as f: f.write(data) if ext in ['png'] : imgurl = os.path.relpath(out, os.path.dirname(e.filename)) print('url: %r' % imgurl) text += '\n <img class="output" src="%s"/> \n\n' % (imgurl) text += '\n\n' return text
def get_object_tree(po: PlacedObject, levels: int = 100, spatial_relations: bool = False, attributes: bool = False) -> str: ss = [] ss.append('%s' % type(po).__name__) d = po.params_to_json_dict() d.pop('children', None) d.pop('spatial_relations', None) if attributes: if d: ds = yaml.safe_dump(d, encoding='utf-8', indent=4, allow_unicode=True, default_flow_style=False) if isinstance(ds, bytes): ds = ds.decode('utf-8') ss.append('\n' + indent(ds, ' ')) if po.children and levels >= 1: ss.append('') N = len(po.children) for i, (child_name, child) in enumerate(po.children.items()): if i != N - 1: prefix1 = u'├ %s ┐ ' % child_name prefix2 = u'│ %s │ ' % (' ' * len(child_name)) else: prefix1 = u'└ %s ┐ ' % child_name prefix2 = u' %s │ ' % (' ' * len(child_name)) c = get_object_tree(child, attributes=attributes, spatial_relations=spatial_relations, levels=levels - 1) sc = indent(c, prefix2, prefix1) n = max(len(_) for _ in sc.split('\n')) sc += '\n' + prefix2[:-2] + u'└' + u'─' * (n - len(prefix2) + 3) ss.append(sc) if spatial_relations: if po.spatial_relations and levels >= 1: ss.append('') for r_name, rel in po.spatial_relations.items(): ss.append( '- from "%s" to "%s" %s ' % (url_from_fqn(rel.a), url_from_fqn(rel.b), rel.transform)) return "\n".join(ss)
def read_one(self, expect_topic=None, timeout=None) -> MsgReceived: try: if expect_topic: waiting_for = f'Expecting topic "{expect_topic}" << {self.nickname}.' else: waiting_for = None msg = read_next_either_json_or_cbor(self.fpout, timeout=timeout, waiting_for=waiting_for) # if self._cc: # msg_b = self._serialize(msg) # self._cc.write(msg_b) # self._cc.flush() topic = msg['topic'] if topic == 'aborted': m = f'I was waiting for a message from component "{self.nickname}" but it aborted with the following error.' m += '\n\n' + indent(msg['data'], '|', f'{self.nickname} error |') raise Exception(m) # XXX if expect_topic: if topic != expect_topic: msg = f'I expected topic "{expect_topic}" but received "{topic}".' raise Exception(msg) # XXX if topic in basic_protocol.outputs: klass = basic_protocol.outputs[topic] else: if self.node_protocol: if topic not in self.node_protocol.outputs: msg = f'Cannot find topic "{topic}" in outputs of detected node protocol.' msg += '\nI know: %s' % sorted(self.node_protocol.outputs) raise Exception(msg) # XXX else: klass = self.node_protocol.outputs[topic] else: if not topic in self.expect_protocol.outputs: msg = f'Cannot find topic "{topic}".' raise Exception(msg) # XXX else: klass = self.expect_protocol.outputs[topic] data = ipce_to_object(msg['data'], {}, expect_type=klass) if self._cc: msg['data'] = object_to_ipce(data, {}, with_schema=True) msg_b = self._serialize(msg) self._cc.write(msg_b) self._cc.flush() timing = ipce_to_object(msg['timing'], {}, expect_type=TimingInfo) self.nreceived += 1 return MsgReceived[klass](topic, data, timing) except StopIteration as e: msg = 'EOF detected on %s after %d messages.' % (self.fnout, self.nreceived) if expect_topic: msg += f' Expected topic "{expect_topic}".' raise StopIteration(msg) from e except TimeoutError as e: msg = 'Timeout detected on %s after %d messages.' % (self.fnout, self.nreceived) if expect_topic: msg += f' Expected topic "{expect_topic}".' raise TimeoutError(msg) from e
def broadcast_event(context, event): import inspect all_handlers = CompmakeGlobalState.EventHandlers.handlers handlers = all_handlers.get(event.name, []) if handlers: for handler in handlers: spec = inspect.getargspec(handler) try: kwargs = {} if 'event' in spec.args: kwargs['event'] = event if 'context' in spec.args: kwargs['context'] = context handler(**kwargs) # TODO: do not catch interrupted, etc. except Exception as e: try: # e = traceback.format_exc(e) msg = [ 'compmake BUG: Error in event handler.', ' event: %s' % event.name, 'handler: %s' % handler, ' kwargs: %s' % list(event.kwargs.keys()), ' bt: ', indent(traceback.format_exc(), '| '), ] msg = "\n".join(msg) CompmakeGlobalState.original_stderr.write(msg) except: pass else: for handler in CompmakeGlobalState.EventHandlers.fallback: handler(context=context, event=event)
def broadcast_event(context, event): import inspect all_handlers = CompmakeGlobalState.EventHandlers.handlers handlers = all_handlers.get(event.name, []) if handlers: for handler in handlers: spec = inspect.getargspec(handler) try: kwargs = {} if 'event' in spec.args: kwargs['event'] = event if 'context' in spec.args: kwargs['context'] = context handler(**kwargs) # TODO: do not catch interrupted, etc. except Exception as e: try: # e = traceback.format_exc(e) msg = [ 'compmake BUG: Error in event handler.', ' event: %s' % event.name, 'handler: %s' % handler, ' kwargs: %s' % list(event.kwargs.keys()), ' bt: ', indent(my_format_exc(e), '| '), ] msg = "\n".join(msg) CompmakeGlobalState.original_stderr.write(msg) except: pass else: for handler in CompmakeGlobalState.EventHandlers.fallback: handler(context=context, event=event)
def check_job_cache_says_failed(job_id, db, e): """ Raises CompmakeBug if the job is not marked as failed. """ if not job_cache_exists(job_id, db): msg = ('The job %r was reported as failed but no record of ' 'it was found.' % job_id) msg += '\n' + 'JobFailed exception:' msg += '\n' + indent(str(e), "| ") raise CompmakeBug(msg) else: cache = get_job_cache(job_id, db) if not cache.state == Cache.FAILED: msg = ('The job %r was reported as failed but it was ' 'not marked as such in the DB.' % job_id) msg += '\n seen state: %s ' % Cache.state2desc[cache.state] msg += '\n' + 'JobFailed exception:' msg += '\n' + indent(str(e), "| ") raise CompmakeBug(msg)
def job_failed(context, event): # @UnusedVariable job_id = event.kwargs['job_id'] reason = event.kwargs['reason'] bt = event.kwargs['bt'] msg = 'Job %r failed:' % job_id # s = reason.strip # if get_compmake_config('echo'): # s += '\n' + bt msg += '\n' + indent(reason.strip(), '| ') if get_compmake_config('echo'): msg += '\n' + indent(bt.strip(), '> ') else: msg += '\nUse "config echo 1" to have errors displayed.' msg += '\nWrite "details %s" to inspect the error.' % job_id error(my_prefix + msg)
def job_failed(context, event): # @UnusedVariable job_id = event.kwargs['job_id'] reason = event.kwargs['reason'] bt = event.kwargs['bt'] msg = 'Job %r failed:' % job_id # s = reason.strip # if get_compmake_config('echo'): # s += '\n' + bt msg += '\n' + indent(reason.strip(), '| ') if get_compmake_config('echo'): s = bt.strip() msg += '\n' + indent(s, '> ') else: msg += '\nUse "config echo 1" to have errors displayed.' msg += '\nWrite "details %s" to inspect the error.' % job_id error(my_prefix + msg)
def make_server_request(token, endpoint, data=None, method='GET', timeout=3): """ Raise RequestFailed or ConnectionError. Returns the result in 'result'. """ server = get_duckietown_server_url() url = server + endpoint headers = {'X-Messaging-Token': token} if data is not None: data = json.dumps(data) req = urllib2.Request(url, headers=headers, data=data) req.get_method = lambda: method try: res = urllib2.urlopen(req, timeout=timeout) data = res.read() except urllib2.URLError as e: msg = 'Cannot connect to server %s' % url raise_wrapped(ConnectionError, e, msg) raise try: result = json.loads(data) except ValueError as e: msg = 'Cannot read answer from server.' msg += '\n\n' + indent(data, ' > ') raise_wrapped(ConnectionError, e, msg) raise if not isinstance(result, dict) or 'ok' not in result: msg = 'Server provided invalid JSON response. Expected a dict with "ok" in it.' msg += '\n\n' + indent(data, ' > ') raise ConnectionError(msg) if result['ok']: if 'result' not in result: msg = 'Server provided invalid JSON response. Expected a field "result".' msg += '\n\n' + indent(result, ' > ') raise ConnectionError(msg) return result['result'] else: msg = 'Failed request for %s:\n%s' % (url, result.get('error', result)) raise RequestFailed(msg)
def instance(self, block_type, name, config, library=None): from procgraph.core.exceptions import ModelInstantionError, SemanticError if library is None: library = self generator = self.get_generator_for_block_type(block_type) try: block = generator(name=name, config=config, library=library) except Exception as e: msg = 'Could not instance block from generator.\n' msg += ' name: %s \n' % name msg += ' config: %s \n' % config msg += ' generator: %s \n' % generator msg += ' of type: %s \n' % describe_type(generator) msg += 'Because of this exception:\n' if isinstance(e, (SemanticError, ModelInstantionError)): msg += indent('%s' % e, '| ') else: msg += indent('%s\n%s' % (e, traceback.format_exc(e)), '| ') raise ModelInstantionError(msg) # TODO: use Procgraph own exception block.__dict__['generator'] = generator return block
def instance(self, block_type, name, config, library=None): from procgraph.core.exceptions import ModelInstantionError, SemanticError if library is None: library = self generator = self.get_generator_for_block_type(block_type) try: block = generator(name=name, config=config, library=library) except Exception as e: msg = 'Could not instance block from generator.\n' msg += ' name: %s \n' % name msg += ' config: %s \n' % config msg += ' generator: %s \n' % generator msg += ' of type: %s \n' % describe_type(generator) msg += 'Because of this exception:\n' if isinstance(e, (SemanticError, ModelInstantionError)): msg += indent('%s' % e, '| ') else: msg += indent('%s\n%s' % (e, traceback.format_exc(e)), '| ') raise ModelInstantionError( msg) # TODO: use Procgraph own exception block.__dict__['generator'] = generator return block
def describe(ip: InteractionProtocol): s = "InteractionProtocol" s += '\n\n' + '* Description:' s += '\n\n' + indent(ip.description.strip(), ' ') s += '\n\n' + '* Inputs:' for name, type_ in ip.inputs.items(): s += '\n %25s: %s' % (name, type_) s += '\n\n' + '* Outputs:' for name, type_ in ip.outputs.items(): s += '\n %25s: %s' % (name, type_) s += '\n\n' + '* Language:' s += '\n\n' + ip.language return s
def interpret_commands_wrap(commands, context, cq): """ Returns None or raises CommandFailed, ShellExitRequested, CompmakeBug, KeyboardInterrupt. """ assert context is not None publish(context, 'command-line-starting', command=commands) try: interpret_commands(commands, context=context, cq=cq) publish(context, 'command-line-succeeded', command=commands) except CompmakeBug: raise except UserError as e: publish(context, 'command-line-failed', command=commands, reason=e) raise CommandFailed(str(e)) except CommandFailed as e: publish(context, 'command-line-failed', command=commands, reason=e) raise except (KeyboardInterrupt, JobInterrupted) as e: publish(context, 'command-line-interrupted', command=commands, reason='KeyboardInterrupt') # If debugging # tb = traceback.format_exc() # print tb # XXX raise CommandFailed(str(e)) # raise CommandFailed('Execution of %r interrupted.' % commands) except ShellExitRequested: raise except Exception as e: tb = traceback.format_exc() msg0 = ('Warning, I got this exception, while it should ' 'have been filtered out already. ' 'This is a compmake BUG that should be reported ' 'at http://github.com/AndreaCensi/compmake/issues') msg = msg0 + "\n" + indent(tb, 'bug| ') publish(context, 'compmake-bug', user_msg=msg, dev_msg="") # XXX raise_wrapped(CompmakeBug, e, msg)
def interpret_commands_wrap(commands, context, cq): """ Returns None or raises CommandFailed, ShellExitRequested, CompmakeBug, KeyboardInterrupt. """ assert context is not None publish(context, 'command-line-starting', command=commands) try: interpret_commands(commands, context=context, cq=cq) publish(context, 'command-line-succeeded', command=commands) except CompmakeBug: raise except UserError as e: publish(context, 'command-line-failed', command=commands, reason=e) raise CommandFailed(str(e)) except CommandFailed as e: publish(context, 'command-line-failed', command=commands, reason=e) raise except (KeyboardInterrupt, JobInterrupted) as e: publish(context, 'command-line-interrupted', command=commands, reason='KeyboardInterrupt') # If debugging # tb = traceback.format_exc() # print tb # XXX raise CommandFailed(str(e)) # raise CommandFailed('Execution of %r interrupted.' % commands) except ShellExitRequested: raise except Exception as e: tb = traceback.format_exc() msg0 = ('Warning, I got this exception, while it should ' 'have been filtered out already. ' 'This is a compmake BUG that should be reported ' 'at http://github.com/AndreaCensi/compmake/issues') msg = msg0 + "\n" + indent(tb, 'bug| ') publish(context, 'compmake-bug', user_msg=msg, dev_msg="") # XXX raise_wrapped(CompmakeBug, e, msg0)
def identify_command(command) -> NodeInfo: d = [{ 'topic': 'wrapper.describe_protocol' }, { 'topic': 'wrapper.describe_config' }, { 'topic': 'wrapper.describe_node' }, { 'topic': 'wrapper.describe_build' }] to_send = b'' for p in d: p['compat'] = ['aido2'] to_send += (json.dumps(p) + '\n').encode('utf-8') cp = subprocess.run(command, input=to_send, capture_output=True) s = cp.stderr.decode('utf-8') sys.stderr.write(indent(s.strip(), '|', ' stderr: |') + '\n\n') # noinspection PyTypeChecker f = BufferedReader(BytesIO(cp.stdout)) stream = read_cbor_or_json_objects(f) res = stream.__next__() pd: ProtocolDescription = ipce_to_object(res['data'], {}, {}, expect_type=ProtocolDescription) res = stream.__next__() cd: ConfigDescription = ipce_to_object(res['data'], {}, {}, expect_type=ConfigDescription) res = stream.__next__() nd: NodeDescription = ipce_to_object(res['data'], {}, {}, expect_type=NodeDescription) res = stream.__next__() bd: BuildDescription = ipce_to_object(res['data'], {}, {}, expect_type=BuildDescription) return NodeInfo(pd, nd, bd, cd)
def __str__(self): s = 'Host %r failed for %r: %s\n%s' % (self.host, self.job_id, self.reason, indent(self.bt, '|')) return s
def render_complete(library, s, raise_errors, realpath, generate_pdf=False, check_refs=False, use_mathjax=True, filter_soup=None, symbols=None, res=None, location=None, ignore_ref_errors=False): """ Transforms markdown into html and then renders the mcdp snippets inside. s: a markdown string with embedded html snippets Returns an HTML string; not a complete document. filter_soup(library, soup) """ if res is None: res = AugmentedResult() if location is None: location = LocationUnknown() from mcdp_report.gg_utils import resolve_references_to_images s0 = s unique = get_md5(realpath)[:8] check_good_use_of_special_paragraphs(s0, res, location) raise_missing_image_errors = raise_errors # Imports here because of circular dependencies from .latex.latex_preprocess import extract_maths, extract_tabular from .latex.latex_preprocess import latex_preprocessing from .latex.latex_preprocess import replace_equations from .macro_col2 import col_macros, col_macros_prepare_before_markdown from .mark.markd import render_markdown from .preliminary_checks import do_preliminary_checks_and_fixes from .prerender_math import prerender_mathjax if isinstance(s, unicode): msg = 'I expect a str encoded with utf-8, not unicode.' raise_desc(TypeError, msg, s=s) # need to do this before do_preliminary_checks_and_fixes # because of & char s, tabulars = extract_tabular(s) s = do_preliminary_checks_and_fixes(s, res, location) # put back tabular, because extract_maths needs to grab them for k, v in tabulars.items(): assert k in s s = s.replace(k, v) # copy all math content, # between $$ and $$ # between various limiters etc. # returns a dict(string, substitution) s, maths = extract_maths(s) # print('maths = %s' % maths) for k, v in list(maths.items()): if v[0] == '$' and v[1] != '$$': if '\n\n' in v: msg = 'The Markdown pre-processor got confused by this math fragment:' msg += '\n\n' + indent(v, ' > ') res.note_error(msg, location) maths[k] = 'ERROR' s = latex_preprocessing(s) s = '<div style="display:none">Because of mathjax bug</div>\n\n\n' + s # cannot parse html before markdown, because md will take # invalid html, (in particular '$ ciao <ciao>' and make it work) s = s.replace('*}', '\*}') s, mcdpenvs = protect_my_envs(s) # print('mcdpenvs = %s' % maths) s = col_macros_prepare_before_markdown(s) # print(indent(s, 'before markdown | ')) s = render_markdown(s) # print(indent(s, 'after markdown | ')) for k, v in maths.items(): if not k in s: msg = 'Internal error while dealing with Latex math.' msg += '\nCannot find %r (= %r)' % (k, v) res.note_error(msg, location) # raise_desc(DPInternalError, msg, s=s) continue def preprocess_equations(x): # this gets mathjax confused x = x.replace('>', '\\gt{}') # need brace; think a<b -> a\lt{}b x = x.replace('<', '\\lt{}') # print('replaced equation %r by %r ' % (x0, x)) return x v = preprocess_equations(v) s = s.replace(k, v) s = replace_equations(s) s = s.replace('\\*}', '*}') # this parses the XML soup = bs(s) other_abbrevs(soup, res, location) substitute_special_paragraphs(soup, res, location) create_notes_from_elements(soup, res, location, unique) # need to process tabular before mathjax escape_for_mathjax(soup) # print(indent(s, 'before prerender_mathjax | ')) # mathjax must be after markdown because of code blocks using "$" s = to_html_stripping_fragment(soup) if use_mathjax: s = prerender_mathjax(s, symbols, res) soup = bs(s) escape_for_mathjax_back(soup) s = to_html_stripping_fragment(soup) # print(indent(s, 'after prerender_mathjax | ')) for k, v in mcdpenvs.items(): # there is this case: # ~~~ # <pre> </pre> # ~~~ s = s.replace(k, v) s = s.replace('<p>DRAFT</p>', '<div class="draft">') s = s.replace('<p>/DRAFT</p>', '</div>') soup = bs(s) mark_console_pres(soup, res, location) # try: # except Exception as e: # msg = 'I got an error while substituting github: references.' # msg += '\nI will ignore this error because it might not be the fault of the writer.' # msg += '\n\n' + indent(str(e), '|', ' error: |') # # must be before make_figure_from_figureid_attr() display_files(soup, defaults={}, res=res, location=location, raise_errors=raise_errors) make_figure_from_figureid_attr(soup, res, location) col_macros(soup) fix_subfig_references(soup) library = get_library_from_document(soup, default_library=library) from .highlight import html_interpret html_interpret(library, soup, generate_pdf=generate_pdf, raise_errors=raise_errors, realpath=realpath, res=res, location=location) if filter_soup is not None: filter_soup(library=library, soup=soup) if False: embed_images_from_library2(soup=soup, library=library, raise_errors=raise_missing_image_errors, res=res, location=location) else: resolve_references_to_images(soup=soup, library=library, raise_errors=raise_missing_image_errors, res=res, location=location) make_videos(soup, res, location, raise_on_errors=False) if check_refs: check_if_any_href_is_invalid(soup, res, location, ignore_ref_errors=ignore_ref_errors) if False: if getuser() == 'andrea': if MCDPConstants.preprocess_style_using_less: run_lessc(soup) else: logger.warning( 'preprocess_style_using_less=False might break the manual') fix_validation_problems(soup) strip_pre(soup) if MCDPManualConstants.enable_syntax_higlighting: syntax_highlighting(soup) if MCDPManualConstants.enforce_lang_attribute: check_lang_codes(soup, res, location) # Fixes the IDs (adding 'sec:'); add IDs to missing ones globally_unique_id_part = 'autoid-DO-NOT-USE-THIS-VERY-UNSTABLE-LINK-' + get_md5(realpath)[:8] fix_ids_and_add_missing(soup, globally_unique_id_part, res, location) check_no_patently_wrong_links(soup, res, location) if MCDPManualConstants.enforce_status_attribute: check_status_codes(soup, realpath, res, location) s = to_html_stripping_fragment(soup) s = replace_macros(s) return s
def write_errors_and_warnings_files(aug, d): if aug.has_result(): id2filename = aug.get_result() else: id2filename = {} # print('id2filename: %s' % sorted(id2filename)) assert isinstance(aug, AugmentedResult) aug.update_refs(id2filename) header = get_notes_panel(aug) manifest = [] nwarnings = len(aug.get_notes_by_tag(MCDPManualConstants.NOTE_TAG_WARNING)) fn = os.path.join(d, 'warnings.html') html = html_list_of_notes(aug, MCDPManualConstants.NOTE_TAG_WARNING, 'warnings', 'warning', header=header) # update_refs_('warnings', html, id2filename) write_data_to_file(str(html), fn, quiet=True) if nwarnings: manifest.append( dict(display='%d warnings' % nwarnings, filename='warnings.html')) msg = 'There were %d warnings: %s' % (nwarnings, fn) logger.warn(msg) ntasks = len(aug.get_notes_by_tag(MCDPManualConstants.NOTE_TAG_TASK)) fn = os.path.join(d, 'tasks.html') html = html_list_of_notes(aug, MCDPManualConstants.NOTE_TAG_TASK, 'tasks', 'task', header=header) # update_refs_('tasks', html, id2filename) write_data_to_file(str(html), fn, quiet=True) if nwarnings: manifest.append( dict(display='%d tasks' % ntasks, filename='tasks.html')) msg = 'There are %d open tasks: %s' % (ntasks, fn) logger.info(msg) nerrors = len(aug.get_notes_by_tag(MCDPManualConstants.NOTE_TAG_ERROR)) fn = os.path.join(d, 'errors.html') html = html_list_of_notes(aug, MCDPManualConstants.NOTE_TAG_ERROR, 'errors', 'error', header=header) # update_refs_('tasks', html, id2filename) write_data_to_file(str(html), fn, quiet=True) if nerrors: manifest.append( dict(display='%d errors' % nerrors, filename='errors.html')) msg = 'I am sorry to say that there were %d errors.\n\nPlease see: %s' % ( nerrors, fn) logger.error('\n\n\n' + indent(msg, ' ' * 15) + '\n\n') fn = os.path.join(d, 'errors_and_warnings.manifest.yaml') write_data_to_file(yaml.dump(manifest), fn, quiet=False) fn = os.path.join(d, 'errors_and_warnings.pickle') res = AugmentedResult() res.merge(aug) write_data_to_file(pickle.dumps(res), fn, quiet=False)
def check_if_any_href_is_invalid(soup, res, location0, extra_refs=None, ignore_ref_errors=False): """ Checks if references are invalid and tries to correct them. also works the magic """ if extra_refs is None: extra_refs = Tag(name='div') else: print('using extra cross refs') # let's first find all the IDs id2element_current, duplicates = get_id2element(soup, 'id') id2element_extra, _ = get_id2element(extra_refs, 'id') id2element = {} id2element.update(id2element_extra) id2element.update(id2element_current) for a in soup.select('[href^="#"]'): href = a['href'] assert href.startswith('#') ID = href[1:] if a.has_attr('class') and "mjx-svg-href" in a['class']: msg = 'Invalid math reference (sorry, no details): href = %s .' % href location = HTMLIDLocation.for_element(a, location0) res.note_error(msg, location) continue if ID not in id2element: # try to fix it # if there is already a prefix, remove it if ':' in href: i = href.index(':') core = href[i + 1:] else: core = ID possible = MCDPManualConstants.all_possible_prefixes_that_can_be_implied matches = [] others = [] for possible_prefix in possible: why_not = possible_prefix + ':' + core others.append(why_not) if why_not in id2element: matches.append(why_not) if len(matches) > 1: msg = '%s not found, and multiple matches for heuristics (%s)' % (href, matches) location = HTMLIDLocation.for_element(a, location0) res.note_error(msg, location) elif len(matches) == 1: # if 'base_url' in element.attrs: # a['href'] = element.attrs['base_url'] + '#' + matches[0] # else: a.attrs['href'] = '#' + matches[0] if matches[0] not in id2element_current: element = id2element[matches[0]] # msg = 'Using foreign resolve for %s -> %s' % (matches[0], a['href']) # logger.info(msg) a.attrs['href_external'] = element.attrs['base_url'] + '#' + matches[0] if show_debug_message_for_corrected_links: msg = '%s not found, but corrected in %s' % (href, matches[0]) location = HTMLIDLocation.for_element(a, location0) res.note_warning(msg, location) else: if has_class(a, MCDPConstants.CLASS_IGNORE_IF_NOT_EXISTENT): del a.attrs['href'] # logger.warning('ignoring link %s' % a) else: msg = 'I do not know what is indicated by the link %r.' % href marker = Tag(name='span') marker.attrs['class'] = 'inside-unknown-link' marker.append(' (unknown ref %s)' % core) a.append(marker) location = HTMLIDLocation.for_element(a, location0) if ignore_ref_errors: msg2 = 'I will ignore this error because this is the first pass:'******'\n\n' + indent(msg, ' > ') res.note_warning(msg2, location) else: res.note_error(msg, location) if ID in duplicates: msg = 'More than one element matching %r.' % href location = HTMLIDLocation.for_element(a, location0) res.note_error(msg, location)
def process(self): """ Start processing jobs. """ # logger.info('Started job manager with %d jobs.' % (len(self.todo))) self.check_invariants() if not self.todo and not self.ready_todo: publish(self.context, 'manager-succeeded', nothing_to_do=True, targets=self.targets, done=self.done, all_targets=self.all_targets, todo=self.todo, failed=self.failed, blocked=self.blocked, ready=self.ready_todo, processing=self.processing) return True publish(self.context, 'manager-phase', phase='init') self.process_init() publish(self.context, 'manager-phase', phase='loop') try: i = 0 while self.todo or self.ready_todo or self.processing: self.log(indent(self._get_situation_string(), '%s: ' % i)) i += 1 self.check_invariants() # either something ready to do, or something doing # otherwise, we are completely blocked if (not self.ready_todo) and (not self.processing): msg = ('Nothing ready to do, and nothing cooking. ' 'This probably means that the Compmake job ' 'database was inconsistent. ' 'This might happen if the job creation is ' 'interrupted. Use the command "check-consistency" ' 'to check the database consistency.\n' + self._get_situation_string()) raise CompmakeBug(msg) self.publish_progress() waiting_on = self.instance_some_jobs() # self.publish_progress() publish(self.context, 'manager-wait', reasons=waiting_on) if self.ready_todo and not self.processing: # We time out as there are no resources publish(self.context, 'manager-phase', phase='wait') self.loop_until_something_finishes() self.check_invariants() self.log(indent(self._get_situation_string(), 'ending: ')) # end while assert not self.todo assert not self.ready_todo assert not self.processing self.check_invariants() self.publish_progress() self.process_finished() publish(self.context, 'manager-succeeded', nothing_to_do=False, targets=self.targets, done=self.done, all_targets=self.all_targets, todo=self.todo, failed=self.failed, ready=self.ready_todo, blocked=self.blocked, processing=self.processing) return True except JobInterrupted as e: from ..ui import error error('Received JobInterrupted: %s' % e) raise except KeyboardInterrupt: raise KeyboardInterrupt('Manager interrupted.') finally: self.cleanup()
def get_cross_refs(src_dirs, permalink_prefix, extra_crossrefs, ignore=[]): res = AugmentedResult() files = look_for_files(src_dirs, "crossref.html") id2file = {} soup = Tag(name='div') def add_from_soup(s, f, ignore_alread_present, ignore_if_conflict): for img in list(s.find_all('img')): img.extract() for e in s.select('[base_url]'): e['external_crossref_file'] = f # Remove the ones with the same base_url for e in list(s.select('[base_url]')): if e.attrs['base_url'] == permalink_prefix: e.extract() for e in s.select('[id]'): id_ = e.attrs['id'] if id_ == 'container': continue # XXX: if id_ in id2file: if not ignore_alread_present: msg = 'Found two elements with same ID "%s":' % id_ msg += '\n %s' % id2file[id_] msg += '\n %s' % f res.note_error(msg) else: id2file[id_] = f e2 = e.__copy__() if ignore_if_conflict: e2.attrs['ignore_if_conflict'] = '1' soup.append(e2) soup.append('\n') ignore = [os.path.realpath(_) for _ in ignore] for _f in files: if os.path.realpath(_f) in ignore: msg = 'Ignoring file %r' % _f logger.info(msg) continue logger.info('cross ref file %s' % _f) data = open(_f).read() if permalink_prefix in data: msg = 'skipping own file' logger.debug(msg) continue s = bs(data) add_from_soup(s, _f, ignore_alread_present=False, ignore_if_conflict=False) if extra_crossrefs is not None: logger.info('Reading external refs\n%s' % extra_crossrefs) try: r = requests.get(extra_crossrefs) except Exception as ex: msg = 'Could not read external cross reference links' msg += '\n %s' % extra_crossrefs msg += '\n\n' + indent(str(ex), ' > ') res.note_error(msg) else: logger.debug('%s %s' % (r.status_code, extra_crossrefs)) if r.status_code == 404: msg = 'Could not read external cross refs: %s' % r.status_code msg += '\n url: ' + extra_crossrefs msg += '\n This is normal if you have not pushed this branch yet.' res.note_warning(msg) # logger.error(msg) s = bs(r.text) add_from_soup(s, extra_crossrefs, ignore_alread_present=True, ignore_if_conflict=True) # print soup res.set_result(str(soup)) return res
def pmake_worker(name, job_queue, result_queue, signal_queue, signal_token, write_log=None): if write_log: f = open(write_log, 'w') def log(s): #print('%s: %s' % (name, s)) f.write('%s: ' % name) f.write(s) f.write('\n') f.flush() else: def log(s): print('%s: %s' % (name, s)) pass log('started pmake_worker()') signal.signal(signal.SIGINT, signal.SIG_IGN) def put_result(x): log('putting result in result_queue..') result_queue.put(x, block=True) if signal_queue is not None: log('putting result in signal_queue..') signal_queue.put(signal_token, block=True) log('(done)') try: while True: log('Listening for job') try: job = job_queue.get(block=True, timeout=5) except Empty: log('Could not receive anything.') continue if job == PmakeSub.EXIT_TOKEN: log('Received EXIT_TOKEN.') break log('got job: %s' % str(job)) function, arguments = job try: result = function(arguments) except JobFailed as e: log('Job failed, putting notice.') log('result: %s' % str(e)) # debug put_result(e.get_result_dict()) except JobInterrupted as e: log('Job interrupted, putting notice.') put_result(dict(abort=str(e))) # XXX except CompmakeBug as e: # XXX :to finish log('CompmakeBug') put_result(e.get_result_dict()) else: log('result: %s' % str(result)) put_result(result) log('...done.') # except KeyboardInterrupt: pass except BaseException as e: reason = 'aborted because of uncaptured:\n' + indent( traceback.format_exc(), '| ') mye = HostFailed(host="???", job_id="???", reason=reason, bt=traceback.format_exc()) log(str(mye)) put_result(mye.get_result_dict()) except: mye = HostFailed(host="???", job_id="???", reason='Uknown exception (not BaseException)', bt="not available") log(str(mye)) put_result(mye.get_result_dict()) log('(put)') if signal_queue is not None: signal_queue.close() result_queue.close() log('clean exit.')
def job_interrupted(context, event): # @UnusedVariable error(my_prefix + 'Job %r interrupted:\n %s' % (event.kwargs['job_id'], indent(event.kwargs['bt'], '> ')))
def manager_host_failed(context, event): # @UnusedVariable s = 'Host failed for job %s: %s' % (event.job_id, event.reason) s += indent(event.bt.strip(), '| ') error(s)
def pmake_worker(name, job_queue, result_queue, signal_queue, signal_token, write_log=None): if write_log: f = open(write_log, "w") def log(s): f.write("%s: " % name) f.write(s) f.write("\n") f.flush() else: def log(s): pass log("started pmake_worker()") signal.signal(signal.SIGINT, signal.SIG_IGN) def put_result(x): log("putting result in result_queue..") result_queue.put(x, block=True) if signal_queue is not None: log("putting result in signal_queue..") signal_queue.put(signal_token, block=True) log("(done)") try: while True: log("Listening for job") job = job_queue.get(block=True) log("got job: %s" % str(job)) if job == PmakeSub.EXIT_TOKEN: break function, arguments = job try: result = function(arguments) except JobFailed as e: log("Job failed, putting notice.") log("result: %s" % str(e)) # debug put_result(e.get_result_dict()) except JobInterrupted as e: log("Job interrupted, putting notice.") put_result(dict(abort=str(e))) # XXX except CompmakeBug as e: # XXX :to finish log("CompmakeBug") put_result(e.get_result_dict()) else: log("result: %s" % str(result)) put_result(result) log("...done.") # except KeyboardInterrupt: pass except BaseException as e: reason = "aborted because of uncaptured:\n" + indent(traceback.format_exc(e), "| ") mye = HostFailed(host="???", job_id="???", reason=reason, bt=traceback.format_exc(e)) log(str(mye)) put_result(mye.get_result_dict()) except: mye = HostFailed(host="???", job_id="???", reason="Uknown exception (not BaseException)", bt="not available") log(str(mye)) put_result(mye.get_result_dict()) log("(put)") if signal_queue is not None: signal_queue.close() result_queue.close() log("clean exit.")
def parmake_job2_new_process(args): """ Starts the job in a new compmake process. """ (job_id, context) = args compmake_bin = which('compmake') db = context.get_compmake_db() storage = db.basepath # XXX: where = os.path.join(storage, 'parmake_job2_new_process') if not os.path.exists(storage): try: os.makedirs(storage) except: pass out_result = os.path.join(where, '%s.results.pickle' % job_id) out_result = os.path.abspath(out_result) cmd = [compmake_bin, storage] if not all_disabled(): cmd += ['--contracts'] cmd += [ '--status_line_enabled', '0', '--colorize', '0', '-c', 'make_single out_result=%s %s' % (out_result, job_id), ] cwd = os.getcwd() cmd_res = system_cmd_result(cwd, cmd, display_stdout=False, display_stderr=False, raise_on_error=False, capture_keyboard_interrupt=False) ret = cmd_res.ret if ret == CompmakeConstants.RET_CODE_JOB_FAILED: # XXX: msg = 'Job %r failed in external process' % job_id msg += indent(cmd_res.stdout, 'stdout| ') msg += indent(cmd_res.stderr, 'stderr| ') res = safe_pickle_load(out_result) os.unlink(out_result) result_dict_check(res) raise JobFailed.from_dict(res) elif ret != 0: msg = 'Host failed while doing %r' % job_id msg += '\n cmd: %s' % " ".join(cmd) msg += '\n' + indent(cmd_res.stdout, 'stdout| ') msg += '\n' + indent(cmd_res.stderr, 'stderr| ') raise CompmakeBug(msg) # XXX: res = safe_pickle_load(out_result) os.unlink(out_result) result_dict_check(res) return res