def __str__(self): s = type(self).__name__ if self.msg: s += "\n\n" + indent(self.msg, ' ') + '\n' else: s += '\n\n (No messages given) \n' if not self.locations: s += '\n(No locations provided)' elif len(self.locations) == 1: s += str(list(self.locations.values())[0]) else: s += '\nThese are the locations indicated:\n' locations = OrderedDict() from mcdp_docs.location import LocationUnknown for k, v in self.locations.items(): if isinstance(v, LocationUnknown): locations[k] = '(location unknown)' else: locations[k] = v s += '\n' + indent(pretty_print_dict(locations), ' ') if self.created_function: s += '\n\nCreated by function %s()' % self.created_function s += '\n in module %s' % self.created_module # s += '\n in file %s' % self.created_file # TODO: use Location if self.prefix: p = "/".join(self.prefix) s = indent(s, p + '> ') return s
def get(self, timeout=0): # @UnusedVariable if self.already_read: msg = 'Compmake BUG: should not call twice.' raise CompmakeException(msg) self.already_read = True assert self.ready() os.remove(self.retcode) stderr = open(self.stderr, 'r').read() stdout = open(self.stdout, 'r').read() stderr = 'Contents of %s:\n' % self.stderr + stderr stdout = 'Contents of %s:\n' % self.stdout + stdout os.remove(self.stderr) os.remove(self.stdout) if self.ret == 0: return elif self.ret == CompmakeConstants.RET_CODE_JOB_FAILED: msg = 'SGE Job failed (ret: %s)\n' % self.ret msg += indent(stderr, 'err > ') # mark_as_failed(self.job_id, msg, None) error(msg) raise JobFailed(msg) else: # XXX RET_CODE_JOB_FAILED is not honored msg = 'SGE Job failed (ret: %s)\n' % self.ret msg += indent(stderr, 'err > ') error(msg) raise JobFailed(msg)
def create_view_instance(self, s, data): try: s.validate(data) except NotValid as e: msg = 'Could not create view instance because the data is not valid ' msg += 'according to the schema.' msg += '\n' + indent(s.__str__(), 'schema: ') msg += '\n' + indent(yaml_dump(data), 'data: ') raise_wrapped(NotValid, e, msg, compact=True) if s in self.s2baseclass: use = self.s2baseclass[s] class Base(use): pass else: class Base(object): pass if isinstance(s, SchemaContext): class ViewContext(ViewContext0, Base): pass try: view = ViewContext(view_manager=self, data=data, schema=s) view.init_context() return view except TypeError as e: msg = 'Probably due to a constructor in Base = %s' % (Base) if s in self.s2baseclass: msg += '\n' + str(self.s2baseclass[s]) raise_wrapped(ValueError, e, msg) if isinstance(s, SchemaHash): class ViewHash(ViewHash0, Base): pass view = ViewHash(view_manager=self, data=data, schema=s) view.init_hash() return view if isinstance(s, SchemaList): class ViewList(ViewList0, Base): pass return ViewList(view_manager=self, data=data, schema=s) if isinstance(s, SchemaString): return ViewString(view_manager=self, data=data, schema=s) if isinstance(s, SchemaBytes): return ViewBytes(view_manager=self, data=data, schema=s) if isinstance(s, SchemaDate): return ViewDate(view_manager=self, data=data, schema=s) raise NotImplementedError(type(s))
def check_status_codes(soup, realpath): for h in all_headers(soup): if 'notoc' in h.attrs: continue if STATUS_ATTR in h.attrs: s = h.attrs[STATUS_ATTR] if not s in allowed_statuses: msg = 'Invalid status code %r; expected one of %r' % ( s, allowed_statuses) msg += '\n' + indent(str(h), ' ') note_error2(h, 'syntax error', msg) else: # Only warn for h1 that are not part: if h.name == 'h1' and not 'part:' in h.attrs.get('id', ''): if not 'catkin_ws' in realpath: # let's not worry about the Software repo for now h2 = h.__copy__() h2.attrs.pop('github-blob-url', None) h2.attrs.pop('github-edit-url', None) msg = 'Status not found for this header:\n\n %s' % str(h2) msg += '\n\n in file %s' % realpath msg += '\n\nPlease set the status for all the top-level headers.' msg += '\n\nThe syntax is:\n\n # My section {#SECTIONID status=STATUS}' msg += '\n\nThese are the possible choices for the status:\n' for k, v in allowed_statuses.items(): if k != STATUS_UNKNOWN: msg += '\n' + indent(v, '', '%23s ' % ('status=%s' % k)) note_error2(h, 'missing status', msg) h.attrs[STATUS_ATTR] = STATUS_UNKNOWN
def repr_long(self): r1 = self.dp1.repr_long() r2 = self.dp2.repr_long() s = 'Parallel2 %% %s ⇸ %s' % (self.get_fun_space(), self.get_res_space()) s += '\n' + indent(r1, '. ', first='\ ') s += '\n' + indent(r2, '. ', first='\ ') return s
def disk_event_interpret(disk_rep, disk_event): fs = { DiskEvents.disk_event_group: disk_event_disk_event_group_interpret, DiskEvents.dir_create: disk_event_dir_create_interpret, DiskEvents.dir_rename: disk_event_dir_rename_interpret, DiskEvents.dir_delete: disk_event_dir_delete_interpret, DiskEvents.file_create: disk_event_file_create_interpret, DiskEvents.file_modify: disk_event_file_modify_interpret, DiskEvents.file_delete: disk_event_file_delete_interpret, DiskEvents.file_rename: disk_event_file_rename_interpret, } ename = disk_event['operation'] if not ename in fs: raise NotImplementedError(ename) intf = fs[ename] arguments = disk_event['arguments'] try: logger.info('%s %s' % (ename, arguments)) intf(disk_rep=disk_rep, **arguments) except Exception as e: msg = 'Could not complete the replay of this event: \n' msg += indent(yaml_dump(disk_event), 'disk_event: ') msg += '\nFor this tree:\n' msg += indent((disk_rep.tree()), ' disk_rep: ') from mcdp_hdb.memdataview import InvalidOperation raise_wrapped(InvalidOperation, e, msg)
def import_name(name): ''' Loads the python object with the given name. Note that "name" might be "module.module.name" as well. ''' try: return __import__(name, fromlist=['dummy']) except ImportError as e: # split in (module, name) if we can if '.' in name: tokens = name.split('.') field = tokens[-1] module_name = ".".join(tokens[:-1]) if False: # previous method try: module = __import__(module_name, fromlist=['dummy']) except ImportError as e: msg = ('Cannot load %r (tried also with %r):\n' % (name, module_name)) msg += '\n' + indent( '%s\n%s' % (e, traceback.format_exc(e)), '> ') raise ImportFailure(msg) if not field in module.__dict__: msg = 'No field %r\n' % (field) msg += ' found in %r.' % (module) raise ImportFailure(msg) return module.__dict__[field] else: # other method, don't assume that in "M.x", "M" is a module. # It could be a class as well, and "x" be a staticmethod. try: module = import_name(module_name) except ImportError as e: msg = ('Cannot load %r (tried also with %r):\n' % (name, module_name)) msg += '\n' + indent( '%s\n%s' % (e, traceback.format_exc(e)), '> ') raise ImportFailure(msg) if not field in module.__dict__: msg = 'No field %r\n' % (field) msg += ' found in %r.' % (module) raise ImportFailure(msg) f = module.__dict__[field] # "staticmethod" are not functions but descriptors, we need extra magic if isinstance(f, staticmethod): return f.__get__(module, None) else: return f else: msg = 'Cannot import name %r.' % (name) msg += '\n' + indent(traceback.format_exc(e), '> ') raise_desc(ImportFailure, msg, sys_path=sys.path)
def download_url_to_file(url, filename): logger.info('Download from %s' % (url)) tmp = filename + '.download' cmd = ['wget', '-O', tmp, url] d8n_make_sure_dir_exists(tmp) res = system_cmd_result(cwd='.', cmd=cmd, display_stdout=False, display_stderr=False, raise_on_error=True, write_stdin='', capture_keyboard_interrupt=False, env=None) if not os.path.exists(tmp): msg = 'Downloaded file does not exist but wget did not give any error.' msg += '\n url: %s' % url msg += '\n downloaded to: %s' % tmp msg += '\n' + indent(str(res), ' | ') d = os.path.dirname(tmp) r = system_cmd_result(d, ['ls', '-l'], display_stdout=False, display_stderr=False, raise_on_error=True) msg += '\n Contents of the directory:' msg += '\n' + indent(str(r.stdout), ' | ') raise Exception(msg) os.rename(tmp, filename) logger.info('-> %s' % friendly_path(filename))
def check_translation_diskrep_to_gitrep(disk_rep0, disk_events, disk_rep1, out): # @UnusedVariable if not disk_events: raise ValueError('no disk events') repo = gitrep_from_diskrep(disk_rep0) wd = repo.working_tree_dir readback = diskrep_from_gitrep(repo) assert_diskreps_same(disk_rep0, readback, 'original', 'written back') logger.debug(wd) logger.debug('\n'+indent(readback.tree(), 'read back |')) logger.debug('\n'+indent(yaml_dump(disk_events), 'disk_events|')) commits = [] for disk_event in disk_events: logger.debug(indent(yaml_dump(disk_event), 'disk_event | ')) apply_disk_event_to_filesystem(wd, disk_event, repo=repo) if repo.untracked_files: logger.debug('adding untracked file %r' % repo.untracked_files) repo.index.add(repo.untracked_files) message = yaml_dump(disk_event) who = disk_event['who'] logger.info('who: %s' % who) actor = who['actor'] instance = who.get('instance', None) host = who.get('host', None) author = Actor(actor, instance) committer = Actor(instance, host) commit = repo.index.commit(message, author=author, committer=committer) commits.append(commit) res = {} res['repo'] = repo return res
def describe_tag(tag): check_isinstance(tag, Tag) def c(tag): x = unicode(tag).encode('utf-8') return x s = "This is the tag:" s += '\n\n' s += indent(c(tag), 'tag |') s += '\n\n' + 'This is the tag in context:' + '\n\n' sc = "" if tag.previousSibling is not None: sc += c(tag.previousSibling) else: sc += '<!-- no prev sibling -->' sc += c(tag) if tag.nextSibling is not None: sc += c(tag.next) else: sc += '<!-- no next sibling -->' s += indent(sc, 'tag in context |') return s
def assert_equal_disk_rep(a, b): h1 = a.hash_code() h2 = b.hash_code() if h1 != h2: msg = 'Hash codes differ.' msg += '\n' + indent(a.tree(), 'disk_rep ') msg += '\n' + indent(b.tree(), 'disk_rep_by_tr ') raise Exception(msg)
def png_from_pdf(pdf_data, density): """ Converts a pdf to png with the given density """ d = mkdtemp() try: tmpfile = os.path.join(d, 'file.pdf') with open(tmpfile, 'wb') as f: f.write(pdf_data) out = os.path.join(d, 'file.png') cmd = [ 'convert', '-density', str(density), tmpfile, '-background', 'white', '-alpha','remove', '-alpha','off', ] shave = True if shave: # warnings.warn('Using shave to fix some bug in imagemagic') cmd += ['-shave', '1'] cmd += ['-strip'] cmd += [out] try: res = system_cmd_result(cwd='.', cmd=cmd, display_stdout=False, display_stderr=False, raise_on_error=True) if not os.path.exists(out): msg = "ImageMagick did not fail, but it didn't write the image it promised." msg += "\n"+indent(" ".join(cmd), " invocation: ") msg += "\n"+ indent(res.stdout or "(no output)", '|', 'stdout: |') msg += "\n"+ indent(res.stderr or "(no output)", '|', 'stderr: |') where = 'problematic.pdf' msg += "\n I will copy the problematic pdf file to %s" % where shutil.copy(tmpfile, where) raise CmdException(msg) except CmdException as e: msg = 'I was not able to use Imagemagick to convert an image.' try: version = system_cmd_result(cwd='.', cmd=['convert', '--version'], display_stdout=False, display_stderr=False, raise_on_error=True) msg += '\n ImageMagick "convert" version:' msg += '\n' + indent(version.stdout, ' | ') except: pass raise_wrapped(ConversionError, e, msg, compact=True) r = open(out,'rb').read() return r finally: shutil.rmtree(d)
def import_name(name): ''' Loads the python object with the given name. Note that "name" might be "module.module.name" as well. ''' try: return __import__(name, fromlist=['dummy']) except ImportError as e: # split in (module, name) if we can if '.' in name: tokens = name.split('.') field = tokens[-1] module_name = ".".join(tokens[:-1]) if False: # previous method try: module = __import__(module_name, fromlist=['dummy']) except ImportError as e: msg = ('Cannot load %r (tried also with %r):\n' % (name, module_name)) msg += '\n' + indent('%s\n%s' % (e, traceback.format_exc(e)), '> ') raise ValueError(msg) if not field in module.__dict__: msg = 'No field %r\n' % (field) msg += ' found in %r.' % (module) raise ValueError(msg) return module.__dict__[field] else: # other method, don't assume that in "M.x", "M" is a module. # It could be a class as well, and "x" be a staticmethod. try: module = import_name(module_name) except ImportError as e: msg = ('Cannot load %r (tried also with %r):\n' % (name, module_name)) msg += '\n' + indent('%s\n%s' % (e, traceback.format_exc(e)), '> ') raise ValueError(msg) if not field in module.__dict__: msg = 'No field %r\n' % (field) msg += ' found in %r.' % (module) raise ValueError(msg) f = module.__dict__[field] # "staticmethod" are not functions but descriptors, we need extra magic if isinstance(f, staticmethod): return f.__get__(module, None) else: return f else: msg = 'Cannot import name %r.' % (name) msg += indent(traceback.format_exc(e), '> ') raise ValueError(msg)
def repr_long(self): r1 = self.dp1.repr_long() r2 = self.dp2.repr_long() s1 = 'Series:' s2 = ' %s ⇸ %s' % (self.get_fun_space(), self.get_res_space()) s = s1 + ' % ' + s2 s += '\n' + indent(r1, '. ', first='\ ') s += '\n' + indent(r2, '. ', first='\ ') return s
def another(): s = """ This is the case of unreasonable demands (1 kg of extra payload): $ mcdp-solve -d src/mcdp_data/libraries/examples/example-battery.mcdplib battery "<1 hour, 1.0 kg, 1 W>" """ s2 = tryit(s) print indent(s2, 's2: ') assert '1 hour' in s2
def process(dirname, e): db_view = db_view_from_dirname(dirname) host_cache = HostCache(db_view) e.repo = db_view.repos[e.repo_name] e.shelf = e.repo.shelves[e.shelf_name] e.library = e.shelf.libraries[e.library_name] e.things = e.library.things.child(e.spec_name) subscribed_shelves = get_all_shelves(db_view) e.context = TheContext(host_cache, db_view, subscribed_shelves, e.library_name) e.mcdp_library = e.context.get_library() source = e.things[e.thing_name] t0 = time.clock() try: context = e.context.child() e.mcdp_library.load_spec(e.spec_name, e.thing_name, context=context) error = None error_string = None exc = None except MCDPException as exc: error = type(exc).__name__ error_string = str(exc) finally: cpu = time.clock() - t0 if gives_syntax_error(source): if isinstance(exc, DPSyntaxError): error = None error_string = None else: error = 'Unexpected' error_string = 'Expected DPSyntaxError error, got %s' % type(exc).__name__ error_string += '\n' + indent(error_string, 'obtained > ') elif gives_semantic_error(source): if isinstance(exc, DPSemanticError): error = None error_string = None else: error = 'Unexpected' error_string = 'Expected DPSemanticError error, got %s' % type(exc).__name__ error_string += '\n' + indent(error_string, 'obtained > ') elif gives_not_implemented_error(source): if isinstance(exc, DPNotImplementedError): error = None error_string = None else: error = 'Unexpected' error_string = 'Expected DPNotImplementedError error, got %s' % type(exc).__name__ error_string += '\n' + indent(error_string, 'obtained > ') if error: logger.error(e.id + ' ' + error) return Result(error_type=error, error_string=error_string, cpu=cpu, warnings=0)
def assert_diskreps_same(a, b, a_text='a', b_text='b'): n = max(len(a_text), len(b_text)) a_text = a_text.rjust(n) + ' | ' b_text = b_text.rjust(n) + ' | ' if a.hash_code() != b.hash_code(): msg = 'The two DiskReps are not the same:' msg += '\n' + indent(a.tree(), a_text) msg += '\n' + indent(b.tree(), b_text) raise ValueError(msg)
def format_user(self): s = self.msg if self.where: s += '\n\n' s += indent(warning_format_where(self.where), ' ') else: s += '\n\n(no where)' s += '\n I obtained this warning:\n' s += '\n' + indent(self.warning.format_user(), ' | ') return s
def replay_events(view_manager, db0, events): db0 = deepcopy(db0) for event in events: event_intepret(view_manager, db0, event) msg = '\nAfter playing event:\n' msg += indent(yaml_dump(event), ' event: ') msg += '\nthe DB is:\n' msg += indent(yaml_dump(db0), ' db: ') logger.debug(msg) return db0
def assert_equal_string(s2, s2_expected, original=None): if s2 != s2_expected: l0 = ' original |' l1 = ' obtained |' l2 = ' expected |' msg = 'Violation:' if original is not None: msg += '\n\n'+indent(make_chars_visible(original), l0) msg += '\n\n'+indent(make_chars_visible(s2), l1) msg += '\n\n'+indent(make_chars_visible(s2_expected), l2) raise ValueError(msg)
def disk_events_from_data_event(disk_map, schema, data_rep, data_event): viewmanager = ViewManager(schema) view = viewmanager.create_view_instance(schema, data_rep) view._schema.validate(data_rep) # As a preliminary check, we check whether this change happened # inside a YAML representation. inside_yaml, name_of_yaml = change_was_inside_YAML(view=view, data_event=data_event, disk_map=disk_map) # If yes, then the result will be a file_modify event for the YAML file. if inside_yaml: return disk_events_from_data_event_inside_yaml(disk_map, data_event, view, p=name_of_yaml) handlers = { DataEvents.leaf_set: disk_events_from_leaf_set, DataEvents.hash_set: disk_events_from_hash_set, DataEvents.dict_setitem: disk_events_from_dict_setitem, DataEvents.dict_delitem: disk_events_from_dict_delitem, DataEvents.dict_rename: disk_events_from_dict_rename, DataEvents.list_append: disk_events_from_list_append, DataEvents.list_delete: disk_events_from_list_delete, DataEvents.list_insert: disk_events_from_list_insert, DataEvents.list_remove: disk_events_from_list_remove, } operation = data_event['operation'] if operation in handlers: f = handlers[operation] arguments = data_event['arguments'] who = data_event['who'] try: evs = f(disk_map=disk_map, view=view, _id='tmp-id', who=who, **arguments) _id = data_event['id'] for i, ev in enumerate(evs): ev['id'] = _id + '-translated-%d' % i return evs except Exception as e: msg = 'Could not succesfully translate using %r:' % f.__name__ msg += '\n' + 'Schema: ' + '\n' + indent(schema, ' schema ') msg += '\n' + 'Data event: ' + '\n' + indent( yaml_dump(data_event), ' data_event ') raise_wrapped(Exception, e, msg) else: raise NotImplementedError(operation)
def get_empty_links_to_fragment(element_to_modify, extra_refs, res): """ Find all empty links that have a reference to a fragment. yield LinkElement """ # logger.debug('building index') # first find all elements by id id2element_local, duplicates = get_id2element(element_to_modify, 'id') id2element_extra, duplicates2 = get_id2element(extra_refs, 'id') for k in id2element_extra: if k in id2element_local: if 'ignore_if_conflict' in id2element_extra[k].attrs: continue msg = 'ID %s in cross references also contained locally.' % k def cut(x): if len(x) < 500: return x else: return x[:500] + ' ... ' msg += '\n\n' + indent(cut(id2element_local[k]), '', 'local: ') msg += '\n\n' + indent(cut(id2element_extra[k]), '', 'crossrefs: ') res.note_error(msg, HTMLIDLocation.for_element(id2element_local[k])) logger.error(msg) id2element = {} id2element.update(id2element_extra) id2element.update(id2element_local) # logger.debug('building index done') for element in get_empty_links(element_to_modify): if not 'href' in element.attrs: continue href = element.attrs['href'] if not href.startswith('#'): continue rest = href[1:] eid = rest query = None linked = id2element.get(eid, None) # noinspection PyArgumentList yield LinkElement(linker=element, eid=eid, linked=linked, query=query)
def assert_data_equal(schema, data1, data2): # @UnusedVariable ''' Checks that two datas are the same, by checking the hashcode. Raises ValueError. ''' from .schema import data_hash_code if data_hash_code(data1) != data_hash_code(data2): msg = 'The two datas are different' msg += '\n' msg += indent(yaml_dump(data1), ' data1 ') msg += '\n' msg += indent(yaml_dump(data2), ' data2 ') raise ValueError(msg)
def raise_incorrect_format(msg, schema, data): msg2 = 'Incorrect format:\n' msg2 += indent(msg, ' ') if isinstance(data, str): datas = data else: datas = yaml_dump(data).encode('utf8') MAX = 512 if len(datas) > MAX: datas = datas[:MAX] + ' [truncated]' if False: msg2 += '\nData:\n' msg2 += indent(datas, ' ') # msg2 += 'repr: '+ datas.__repr__() raise_desc(IncorrectFormat, msg2, schema=str(schema))
def __str__(self): msg = ('The command: %s\n' ' in dir: %s\n' % (self.cmd, self.cwd)) if self.interrupted: msg += 'Was interrupted by the user\n' else: msg += 'returned: %s' % self.ret if self.rets is not None: msg += '\n' + indent(self.rets, 'error>') if self.stdout: msg += '\n' + indent(self.stdout, 'stdout>') if self.stderr: msg += '\n' + indent(self.stderr, 'stderr>') return msg
def png_error_catch2(self, request, func): """ func is supposed to return an image response. If it raises an exception, we create an image with the error and then we add the exception to the list of exceptions. """ try: return func() except Exception as e: import traceback s = traceback.format_exc(e) try: logger.error(s) except UnicodeEncodeError: pass s = str(s) url = request.url referrer = request.referrer n = 'Error during rendering an image.' n+= '\n url: %s' % url n+= '\n referrer: %s' % referrer n += '\n' + indent(s, '| ') self.exceptions.append(n) from mcdp_web.utils.image_error_catch_imp import response_image return response_image(request, s)
def gg_get_format(gg, data_format): from reprep import Report r = Report() do_dot = data_format == 'dot' do_png = data_format == 'png' do_pdf = data_format == 'pdf' do_svg = data_format == 'svg' with timeit_wall('gg_figure %s' % data_format): gg_figure(r, 'graph', gg, do_dot=do_dot, do_png=do_png, do_pdf=do_pdf, do_svg=do_svg) if data_format == 'pdf': pdf = r.resolve_url('graph_pdf').get_raw_data() return pdf elif data_format == 'png': png = r.resolve_url('graph/graph').get_raw_data() return png elif data_format == 'dot': dot = r.resolve_url('dot').get_raw_data() return dot elif data_format == 'svg': svg = r.resolve_url('graph_svg').get_raw_data() if '<html>' in svg: msg = 'I did not expect a tag <html> in the SVG output' svg = indent(svg, '> ') raise_desc(Exception, msg, svg=svg) return svg else: raise ValueError('No known format %r.' % data_format)
def choose_best_icon(iconoptions, image_source): """ Returns the name of a file, or raise exception KeyError. """ # logger.debug('Looking for %s in %s.' % (str(iconoptions), imagepaths)) exts = MCDPConstants.exts_for_icons iconoptions = [_ for _ in iconoptions if _ is not None] errors = [] for option in iconoptions: if option is None: continue for ext in exts: try: data = image_source.get_image(option, ext) except NoImageFound as e: errors.append(e) else: temp_file = NamedTemporaryFile(suffix='.' + ext, delete=False) with open(temp_file.name, 'wb') as f: f.write(data) return temp_file.name if len(errors) == 1: raise errors[0] else: msg = 'Could not find any match for options %r' % iconoptions for option, e in zip(iconoptions, errors): msg += '\n' + indent(str(e), ' %s > ' % option) raise NoImageFound(msg)
def _load_next(self): try: try: res = self.iterator.next() if not isinstance(res, tuple): msg = "Expected tuple (signal, timestamp, value), obtained %s" % describe_type(res) raise ValueError(msg) if not len(res) == 3: raise ValueError("Required len 3 tuple; obtained %d." % len(res)) signal, timestamp, value = res except StopIteration: raise except Exception as e: msg = "Could not call next() on user-given iterator.\n" msg += " iterator: %s\n" % str(self.iterator) msg += " of type: %s\n" % describe_type(self.iterator) msg += "because of this error:\n" msg += indent(string.strip("%s\n%s" % (e, traceback.format_exc(e))), "| ") raise ModelExecutionError(msg, self) if not isinstance(signal, (str, int)): msg = "Expected a string or number for the signal, got %s" % describe_value(signal) raise ValueError(msg) if not isinstance(timestamp, float): msg = "Expected a number for the timestamp, got %s" % describe_value(timestamp) raise ValueError(msg) self.next_signal = signal self.next_timestamp = timestamp self.next_value = value self.has_next = True except StopIteration: self.has_next = False
def tree_long(self, n=None): if n is None: n = 120 s = type(self).__name__ u = lambda x: x.decode('utf-8') ulen = lambda x: len(u(x)) # # def clip(x, n): # s = str(x) # unicode_string = s.decode("utf-8") # l = len(unicode_string) # s = s + ' ' * (n - l) # if len(u(s)) > n: # x = u(s) # x = x[:n - 3] + '...' # s = x.encode('utf-8') # return s s2 = "" head = s + ' ' * (n - ulen(s) - ulen(s2)) + s2 for x in self._children(): head += '\n' + indent(x.tree_long(n - 2), ' ') return head
def rosbag_info(bag): """ Returns a dictionary with the fields returned by "rosbag info". """ if not os.path.exists(bag): raise ValueError('no file %r' % bag) warnings.warn('Check exit code') cmd = ['rosbag', 'info', '--yaml', bag] cwd = os.getcwd() # print('cmd: %s' % cmd) res = system_cmd_result(cwd, cmd, display_stdout=False, display_stderr=False, raise_on_error=True, capture_keyboard_interrupt=False) stdout = res.stdout # stdout = subprocess.Popen( # stdout=subprocess.PIPE).communicate()[0] try: info_dict = yaml.load(stdout) except: logger.error('Could not parse yaml:\n%s' % indent(stdout, '| ')) raise return info_dict
def rosbag_info(bag): """ Returns a dictionary with the fields returned by "rosbag info". """ if not os.path.exists(bag): raise ValueError('no file %r' % bag) # raise Exception('rosbag_info for %r' % bag) cmd = ['rosbag', 'info', '--yaml', bag] cwd = os.getcwd() res = system_cmd_result(cwd, cmd, display_stdout=False, display_stderr=False, raise_on_error=True, capture_keyboard_interrupt=False) stdout = res.stdout try: info_dict = yaml.load(stdout) except: logger.error('Could not parse yaml:\n%s' % indent(stdout, '| ')) raise return info_dict
def view_exception(self, exc, request): request.response.status = 500 # Internal Server Error import traceback compact = (DPSemanticError, DPSyntaxError) if isinstance(exc, compact): s = exc.__str__() else: s = traceback.format_exc(exc) url = request.url referrer = request.referrer n = 'Error during serving this URL:' n += '\n url: %s' % url n += '\n referrer: %s' % referrer ss = traceback.format_exc(exc) n += '\n' + indent(ss, '| ') self.exceptions.append(n) if 'library' in request.matchdict: library = self.get_current_library_name(request) url_refresh = '/libraries/%s/refresh_library' % library else: url_refresh = None u = unicode(s, 'utf-8') logger.error(u) return {'exception': u, 'url_refresh': url_refresh}
def check_status_codes(soup, realpath, res, location): allowed_statuses = MCDPManualConstants.allowed_statuses for h in all_headers(soup): if MCDPManualConstants.ATTR_NOTOC in h.attrs: continue if MCDPManualConstants.ATTR_STATUS in h.attrs: s = h.attrs[MCDPManualConstants.ATTR_STATUS] if not s in allowed_statuses: msg = 'Invalid status code %r.\n I expected one of: %s' % ( s, ", ".join(allowed_statuses)) # msg += '\n' + indent(str(h), ' ') res.note_error(msg, HTMLIDLocation.for_element(h, location)) else: # Only warn for h1 that are not part: if h.name == 'h1' and not 'part:' in h.attrs.get('id', ''): if not 'catkin_ws' in realpath: # let's not worry about the Software repo for now h2 = h.__copy__() h2.attrs.pop('github-blob-url', None) h2.attrs.pop(MCDPManualConstants.ATTR_GITHUB_EDIT_URL, None) msg = 'Status not found for this header:\n\n %s' % str(h2) msg += '\n\n in file %s' % realpath msg += '\n\nPlease set the status for all the top-level headers.' msg += '\n\nThe syntax is:\n\n # My section {#SECTIONID status=STATUS}' msg += '\n\nThese are the possible choices for the status:\n' for k, v in allowed_statuses.items(): if k != MCDPManualConstants.STATUS_UNKNOWN: msg += '\n' + indent(v, '', '%23s ' % ('status=%s' % k)) res.note_error(msg, HTMLIDLocation.for_element(h, location)) h.attrs[MCDPManualConstants. ATTR_STATUS] = MCDPManualConstants.STATUS_UNKNOWN
def allowed_(self, privilege, principals): if not privilege in Privileges.ALL_PRIVILEGES: msg = 'Unknown privilege %r' % privilege raise ValueError(msg) # We grant root all privileges if MCDPConstants.ROOT in principals: return True for r in self.rules: matches = ((r.privilege == Privileges.SPECIAL_ALL_WILDCARD) or (r.privilege == privilege)) if not matches: continue if r.to_whom in principals: if r.allow_or_deny == MCDPConstants.ALLOW: return True elif r.allow_or_deny == MCDPConstants.DENY: return False else: assert False, r msg = 'Permission %r denied for %s' % (privilege, principals) msg += '\n' + indent('\n'.join(str(_) for _ in self.rules), ' ') logger_access.debug(msg) return False
def render_dot_root(level, node, children_results): # @UnusedVariable s = "digraph G { \n" s += """ graph [fontname = "FONT"]; node [fontname = "FONT"]; edge [fontname = "FONT"]; """.replace('FONT', 'Anka/Coder Narrow') if self.options: for key, value in self.options.iteritems(): s += (" %s=%s;" % (key, value)) s += ("\n") assert isinstance(s, str), s.__repr__() r = indented_results(children_results) # .decode("unicode_escape") try: s += r except UnicodeDecodeError: # print r[210:220].__repr__() # print type(r), type(s) # print r[1200:1230].__repr__() # print r.__repr__() raise self.fd = StringIO() for n in self.__nodes: self.dotLinks(n) s += indent(self.fd.getvalue(), ' ') s += '}' return s
def summary(results, out, errors_only): s = "" shelf_name = None library_name = None spec_name = None nerrors = 0 from termcolor import colored c1 = lambda s: colored(s, color='cyan') ce = lambda s: colored(s, color='red') c*k = lambda s: colored(s, color='green') cyellow = lambda s: colored(s, color='yellow') cpu = [] for _ in sorted(results): e, result = results[_] cpu.append(result.cpu) if result.error_type is not None: nerrors += 1 if e.shelf_name != shelf_name: shelf_name = e.shelf_name s += c1('\n Shelf %s' % shelf_name) if e.library_name != library_name: library_name = e.library_name s += c1('\n Library %s' % library_name) if e.spec_name != spec_name: spec_name = e.spec_name s += c1('\n %s' % spec_name) if result.error_type: s += ce('\n %20s %s' % (result.error_type, e.thing_name)) s += '\n' + indent(result.error_string[:200], ' > ') else: if not errors_only: if result.warnings: warnings = result.warnings else: warnings = '' ms = 1000 * result.cpu ok = '%dms' % ms if ms > 1000: ctime = cyellow else: ctime = lambda x: x s += ctime('\n %20s' % ok) + ' ' + c*k( '%s %s' % (e.thing_name, warnings)) if result.error_type: fn = os.path.join(out, '%s.txt' % e.id) with open(fn, 'w') as f: f.write(result.error_string) logger.info('wrote %s' % fn) s += '\n' s += '\nNumber of errors: %d' % nerrors s += '\nCPU median: %s ms' % (1000 * np.median(cpu)) print(s) fn = os.path.join(out, 'stats.txt') with open(fn, 'w') as f: f.write(s) logger.info('wrote %s' % fn)
def to_html( self, root, max_levels, ): s = u'' if not root: s += ( u"""<a class="toc_link toc_link-depth-%s number_name toc_a_for_%s" href="#%s"></a>""" % (self.depth, self.header_level, self.id)) # logger.info(str(bs(s))) if max_levels and self.items: s += '<ul class="toc_ul-depth-%s toc_li_for_%s">' % ( self.depth, self.header_level) for item in self.items: sitem = item.to_html(root=False, max_levels=max_levels - 1) sitem = indent(sitem, ' ') s += ( '\n <li class="toc_li-depth-%s toc_ul_for_%s">\n%s\n </li>' % (self.depth, self.header_level, sitem)) s += '\n</ul>' return s
def repr_long(self): s = 'CoProduct %% %s ⇸ %s' % (self.get_fun_space(), self.get_res_space()) for dp in self.dps: r1 = dp.repr_long() s += '\n' + indent(r1, '. ', first='^ ') return s
def document_final_pass_before_toc(soup, remove, remove_selectors, res=None, location=None): if res is None: logger.warn('no res passed') res = AugmentedResult() if location is None: location = LocationUnknown() logger.info('reorganizing contents in <sections>') with timeit('find body'): body = soup.find('body') if body is None: msg = 'Cannot find <body>:\n%s' % indent(str(soup)[:1000], '|') raise ValueError(msg) with timeit('reorganize_contents'): body2 = reorganize_contents(body) process_assignment(body2, res, location) body.replace_with(body2) # Removing stuff with timeit('remove stuff'): do_remove_stuff(body2, remove_selectors, remove) with timeit('move_things_around'): move_things_around(soup=soup, res=res)
def check_missing_connections(context): """ Checks that all resources and functions are connected. """ def xsorted(x): return sorted(x) unconnected_fun, unconnected_res = get_missing_connections(context) s = "" if unconnected_fun: s += "There are some unconnected functions:" for n, fn in xsorted(unconnected_fun): s += '\n- function %r of dp %r' % (fn, n) if False: msg = 'One way to fix this is to add an explicit function:\n' fn2 = 'f' fix = "provides %s [unit]" % fn2 if context.is_new_resource(n): ref = n else: ref = '%s.%s' % (n, fn) fix += '\n' + "%s >= %s" % (ref, fn2) msg += indent(fix, ' ') s += '\n' + indent(msg, 'help: ') if unconnected_res: s += "\nThere are some unconnected resources:" for n, rn in xsorted(unconnected_res): s += '\n- resource %s of dp %r' % (rn, n) if False: msg = 'One way to fix this is to add an explicit resource:\n' rn2 = 'r' fix = "requires %s [unit]" % rn2 if context.is_new_function(n): ref = n else: ref = '%s.%s' % (n, rn) # todo: omit '.' if n is fix += '\n' + "%s >= %s" % (rn2, ref) msg += indent(fix, ' ') s += '\n' + indent(msg, 'help: ') if s: raise NotConnected(s)
def instantiate(function_name, parameters): try: function = import_name(function_name) except ValueError as e: msg = 'instantiate(): Cannot find function or constructor %r:\n' % (function_name) msg += indent('%s' % (e), '> ') raise SemanticMistake(msg) try: # XXX TypeError is too broad, we should bind the params explicitly return function(**parameters) except TypeError as e: params = ', '.join(['%s=%r' % (k, v) for (k, v) in parameters.items()]) msg = ('instantiate(): Could not call function %r\n with params %s:' % (function_name, params)) msg += '\n' + indent('%s\n%s' % (e, traceback.format_exc(e)), '> ') raise SemanticMistake(msg)
def format(self): if not self.chronology: return '(empty)' fs = [] for c in self.chronology: s = c.format() s = indent(s, ' ', first='- ') fs.append(s) return "\n".join(fs)
def rosbag_info(bag): warnings.warn('remove, use the other version') stdout = subprocess.Popen(['rosbag', 'info', '--yaml', bag], stdout=subprocess.PIPE).communicate()[0] try: info_dict = yaml.load(stdout) except: logger.error('Could not parse yaml:\n%s' % indent(stdout, '| ')) raise return info_dict
def repr_long(self): s = "%s[%s]" % (type(self).__name__, len(self.spaces)) for label, S in zip(self.labels, self.spaces): prefix0 = " %s. " % label prefix1 = " " * len(prefix0) s += "\n" + indent(S.repr_long(), prefix1, first=prefix0) if hasattr(S, ATTRIBUTE_NDP_RECURSIVE_NAME): a = getattr(S, ATTRIBUTE_NDP_RECURSIVE_NAME) s += '\n labeled as %s' % a.__str__() return s
def recursive_print(x): if not isnamedtupleinstance(x): return x.__repr__() s = type(x).__name__ s += ':\n' for k, v in x._asdict().items(): if k == 'where': continue first = ' %s: ' % k prefix = ' '* len(first) first += '|' prefix += '|' r = recursive_print(v).strip() s += indent(r, prefix, first=first) s += '\n' if hasattr(s, 'warning'): msg = getattr(s, 'warning') s += '\n' + indent(msg, ' W ') return s
def repr_long(self): s = "%s[%s]" % (type(self).__name__, len(self.subs)) for i, S in enumerate(self.subs): prefix0 = " %d. " % i prefix1 = " " s += "\n" + indent(S.repr_long(), prefix1, first=prefix0) if hasattr(S, ATTRIBUTE_NDP_RECURSIVE_NAME): a = getattr(S, ATTRIBUTE_NDP_RECURSIVE_NAME) s += "\n labeled as %s" % a.__str__() return s
def instantiate_spec(code_spec): ''' code_spec must be a sequence [string, dictionary], giving the python function (or class) to instantiate, along with its parameters. ''' try: function_name = code_spec[0] parameters = code_spec[1] assert isinstance(function_name, str) assert isinstance(parameters, dict) return instantiate(function_name, parameters) except Exception as e: msg = 'Could not instance the spec:\n' msg += indent(format_code_spec(code_spec).strip(), ' ').strip() msg += '\nbecause of this error:\n' if isinstance(e, InstantiationException): st = str(e) else: st = traceback.format_exc(e) msg += indent(st.strip(), '| ') msg = msg.strip() raise InstantiationException(msg)
def instance(self, block_type, name, config, library=None): from procgraph.core.exceptions import ModelInstantionError, SemanticError if library is None: library = self generator = self.get_generator_for_block_type(block_type) try: block = generator(name=name, config=config, library=library) except Exception as e: msg = 'Could not instance block from generator.\n' msg += ' name: %s \n' % name msg += ' config: %s \n' % config msg += ' generator: %s \n' % generator msg += ' of type: %s \n' % describe_type(generator) msg += 'Because of this exception:\n' if isinstance(e, (SemanticError, ModelInstantionError)): msg += indent('%s' % e, '| ') else: msg += indent('%s\n%s' % (e, traceback.format_exc(e)), '| ') raise ModelInstantionError(msg) # TODO: use Procgraph own exception block.__dict__['generator'] = generator return block
def eval_lfunction(lf, context): check_isinstance(context, ModelBuildingContext) constants = (CDP.Collection, CDP.SimpleValue, CDP.SpaceCustomValue, CDP.Top, CDP.Bottom, CDP.Minimals, CDP.Maximals, CDP.NatConstant, CDP.RcompConstant) if isinstance(lf, (CDP.NewFunction, CDP.DerivResourceRef)): msg = 'The functionality %r cannot be used on this side of the constraint.' raise_desc(DPSemanticError, msg % lf.name.value) if isinstance(lf, constants): from mcdp_lang.eval_constant_imp import eval_constant res = eval_constant(lf, context) assert isinstance(res, ValueWithUnits) return get_valuewithunits_as_function(res, context) from .eval_lfunction_imp_maketuple import eval_MakeTuple_as_lfunction from .eval_uncertainty import eval_lfunction_Uncertain from .eval_lfunction_imp_label_index import eval_lfunction_label_index from .eval_lfunction_imp_label_index import eval_lfunction_tupleindexfun cases = { CDP.Function: eval_lfunction_Function, CDP.NewResource: eval_lfunction_newresource, CDP.MakeTuple: eval_MakeTuple_as_lfunction, CDP.UncertainFun: eval_lfunction_Uncertain, CDP.DisambiguationFun: eval_lfunction_disambiguation, CDP.FunctionLabelIndex: eval_lfunction_label_index, CDP.TupleIndexFun: eval_lfunction_tupleindexfun, CDP.AnyOfFun: eval_lfunction_anyoffun, CDP.InvMult: eval_lfunction_invmult, CDP.InvPlus: eval_lfunction_invplus, CDP.VariableRef: eval_lfunction_variableref, CDP.ActualVarRef: eval_lfunction_ActualVarRef, CDP.DerivFunctionRef: eval_lfunction_DerivFunctionRef, CDP.FValueMinusN: eval_lfunction_FValueMinusN, CDP.GenericOperationFun: eval_lfunction_genericoperationfun, CDP.ConstantRef: eval_lfunction_ConstantRef, } for klass, hook in cases.items(): if isinstance(lf, klass): return hook(lf, context) if True: # pragma: no cover r = recursive_print(lf) msg = 'eval_lfunction(): cannot evaluate this as a function:' msg += '\n' + indent(r, ' ') raise_desc(DPInternalError, msg)
def desc(self): s = 'SimpleWrap' from mcdp_library.library import ATTR_LOAD_NAME if hasattr(self, ATTR_LOAD_NAME): s += '\n (loaded as %r)' % getattr(self, ATTR_LOAD_NAME) for f in self.get_fnames(): s += '\n provides %10s (%s) ' % (f, self.get_ftype(f)) for r in self.get_rnames(): s += '\n requires %10s (%s) ' % (r, self.get_rtype(r)) dp = self.get_dp() s += '\n %s' % type(dp) s += '\n' + indent(dp.repr_long(), ' | ') return s
def print_ast(x): try: if isnamedtuplewhere(x): s = '%s' % type(x).__name__ s += ' %r' % x.where for k, v in iterate_sub(x): first = ' %s: ' % k s += '\n' + indent(print_ast(v), ' ' * len(first), first=first) if x.where is None: raise ValueError(x) return s else: return x.__repr__() except ValueError as e: raise_wrapped(ValueError, e, 'wrong', x=x)
def get_plotters(plotters, space): available = [] errors = [] for name, plotter in plotters.items(): try: plotter.check_plot_space(space) available.append((name, plotter)) except NotPlottable as e: errors.append((name, plotter, e)) if available: return available msg = 'Could not find any plotter for space %s.' % space msg += '\n None of these worked: %s' % plotters.keys() msg += '\nTraceback:' for name, plotter, e in errors: msg += '\n%r (%s)\n%s' % (name, type(plotter), indent(str(e), ' ')) raise_desc(NotPlottable, msg)
def check_equal(self, a, b): # first check that these belongs at all for c in [a, b]: if not isinstance(c, tuple) or len(c) != len(self.subs): msg = "Invalid argument is not a tuple." raise_desc(TypeError, msg, argument=c) problems = [] for i, (sub, x, y) in enumerate(zip(self.subs, a, b)): try: sub.check_equal(x, y) except NotEqual as e: msg = "#%d (%s): %s ≰ %s." % (i, sub, x, y) msg += "\n" + indent(str(e).strip(), "| ") problems.append(msg) if problems: msg = "Equal does not hold.\n" msg = "\n".join(problems) raise_desc(NotEqual, msg, args_first=False, self=self, a=a, b=b)
def _recursive_create(self, f): new_values = {} for k, c in self.get_components_and_values(): c2 = f(c) if not isinstance(c2, TypsyType): msg = ('Could not transform %r (%s, %s) using %s: got %s' % (k, c, describe_type(c), f, describe_type(c2))) raise ValueError(msg) new_values[k] = c2 try: return type(self).create_from_kwargs(**new_values) except (Exception, TypeError) as e: msg = 'Could not call create_from_kwargs for %r' % type(self) msg += '\n values: %s' % new_values msg += '\n' + indent(traceback.format_exc(e), '> ') raise Exception(msg)
def __repr__(self): s = 'NamedDPCoproduct' if hasattr(self, ATTR_LOAD_NAME): s += '\n (loaded as %r)' % getattr(self, ATTR_LOAD_NAME) if hasattr(self, ATTRIBUTE_NDP_RECURSIVE_NAME): s += '\n (labeled as %s)' % getattr(self, ATTRIBUTE_NDP_RECURSIVE_NAME).__str__() for f in self.get_fnames(): s += '\n provides %s [%s]' % (f, self.get_ftype(f)) for r in self.get_rnames(): s += '\n requires %s [%s]' % (r, self.get_rtype(r)) for label, ndp in zip(self.labels, self.ndps): prefix = '- %s: ' % label prefix2 = ' ' * len(prefix) s += '\n' + indent(ndp, prefix2, prefix) return s
def check_leq(self, a, b): if do_extra_checks(): self.belongs(a) self.belongs(b) problems = [] if not( isinstance(a, tuple) and len(a) == len(self.subs)): raise ValueError(a) if not( isinstance(b, tuple) and len(b) == len(self.subs)): raise ValueError(b) for i, (sub, x, y) in enumerate(zip(self.subs, a, b)): try: sub.check_leq(x, y) except NotLeq as e: msg = '#%d (%s): %s ≰ %s.' % (i, sub, x, y) msg += '\n' + indent(str(e).strip(), '| ') problems.append(msg) if problems: msg = 'Leq does not hold.\n' msg = "\n".join(problems) raise_desc(NotLeq, msg, self=self, a=a, b=b)
def belongs(self, x): if not isinstance(x, tuple): raise_desc(NotBelongs, "Not a tuple", x=x, self=self) if not len(x) == len(self.subs): raise_desc( NotBelongs, "Length does not match: len(x) = %s != %s" % (len(x), len(self.subs)), x=x, self=self ) problems = [] for i, (sub, xe) in enumerate(zip(self.subs, x)): try: sub.belongs(xe) except NotBelongs as e: msg = "#%s: Component %s does not belong to factor %s" % (i, xe, sub) msg += "\n" + indent(str(e).strip(), "| ") problems.append(msg) if problems: msg = "The point does not belong to this product space.\n" msg += "\n".join(problems) raise_desc(NotBelongs, msg, args_first=False, self=self, x=x)
def get_best_match(opname, rtypes, are_they_constant, generic_ops): """ Raises NotMatching """ check_isinstance(opname, str) assert len(rtypes) == len(are_they_constant) names_available = set(name for _,name,_ in generic_ops) if not opname in names_available: msg = 'Unknown operation %r.' % opname msg += ' Available: ' + ", ".join(sorted(names_available)) + '.' raise_desc(NotMatching, msg) problems = [] for (id_op, name, op) in generic_ops: if name != opname: continue try: symbols = match_op(op, rtypes, are_they_constant) except NotMatching as e: problems.append((id_op, e)) continue except DPSemanticError as e: # for now, warning about floor() not being scott-continuous raise return op, symbols msg = ('Could not find a match with any of the %d version(s) of %r.' % (len(problems), opname)) ops = [] for R, is_constant in zip(rtypes, are_they_constant): o = 'constant %s' % R if is_constant else '%s' % R ops.append(o) proto = "%s(%s)" % (opname, ", ".join(ops)) msg += '\n' + 'I was looking for a prototype like:\n\n %s' % proto msg += '\n\nHowever, I got these problems:\n' for id_op, e in problems: prefix = ' ' + id_op + ':' msg += '\n' + indent(' ' + str(e).strip(), ' ' * len(prefix), first=prefix) raise_desc(NotMatching, msg)