def parse_tolerance(source, want): """ Returns a version of ``want`` marked up with the tolerance tags specified in ``source``. INPUT: - ``source`` -- a string, the source of a doctest - ``want`` -- a string, the desired output of the doctest OUTPUT: - ``want`` if there are no tolerance tags specified; a :class:`MarkedOutput` version otherwise. EXAMPLES:: sage: from sage.doctest.parsing import parse_tolerance sage: marked = parse_tolerance("sage: s.update(abs_tol = .0000001)", "") sage: type(marked) <type 'str'> sage: marked = parse_tolerance("sage: s.update(tol = 0.1); s.rel_tol # abs tol 0.01", "") sage: marked.tol 0 sage: marked.rel_tol 0 sage: marked.abs_tol 0.01 """ safe, literals = strip_string_literals(source) first_line = safe.split('\n', 1)[0] if '#' not in first_line: return want comment = first_line[first_line.find('#')+1:] # strip_string_literals replaces comments comment = literals[comment] if random_marker.search(comment): want = MarkedOutput(want).update(random=True) else: m = tolerance_pattern.search(comment) if m: rel_or_abs, epsilon = m.groups() if epsilon is None: epsilon = 1e-15 else: epsilon = float(epsilon.strip()) if rel_or_abs is None: want = MarkedOutput(want).update(tol=epsilon) elif rel_or_abs.startswith('rel'): want = MarkedOutput(want).update(rel_tol=epsilon) elif rel_or_abs.startswith('abs'): want = MarkedOutput(want).update(abs_tol=epsilon) else: raise RuntimeError return want
def parse_tolerance(source, want): """ Returns a version of ``want`` marked up with the tolerance tags specified in ``source``. INPUT: - ``source`` -- a string, the source of a doctest - ``want`` -- a string, the desired output of the doctest OUTPUT: - ``want`` if there are no tolerance tags specified; a :class:`MarkedOutput` version otherwise. EXAMPLES:: sage: from sage.doctest.parsing import parse_tolerance sage: marked = parse_tolerance("sage: s.update(abs_tol = .0000001)", "") sage: type(marked) <type 'str'> sage: marked = parse_tolerance("sage: s.update(tol = 0.1); s.rel_tol # abs tol 0.01", "") sage: marked.tol 0 sage: marked.rel_tol 0 sage: marked.abs_tol 0.01 """ safe, literals = strip_string_literals(source) first_line = safe.split('\n', 1)[0] if '#' not in first_line: return want comment = first_line[first_line.find('#') + 1:] # strip_string_literals replaces comments comment = literals[comment] if random_marker.search(comment): want = MarkedOutput(want).update(random=True) else: m = tolerance_pattern.search(comment) if m: rel_or_abs, epsilon = m.groups() if epsilon is None: epsilon = 1e-15 else: epsilon = float(epsilon.strip()) if rel_or_abs is None: want = MarkedOutput(want).update(tol=epsilon) elif rel_or_abs.startswith('rel'): want = MarkedOutput(want).update(rel_tol=epsilon) elif rel_or_abs.startswith('abs'): want = MarkedOutput(want).update(abs_tol=epsilon) else: raise RuntimeError return want
def parse_optional_tags(string): """ Returns a set consisting of the optional tags from the following set that occur in a comment on the first line of the input string. - 'long time' - 'not implemented' - 'not tested' - 'known bug' - 'optional: PKG_NAME' -- the set will just contain 'PKG_NAME' EXAMPLES:: sage: from sage.doctest.parsing import parse_optional_tags sage: parse_optional_tags("sage: magma('2 + 2')# optional: magma") set(['magma']) sage: parse_optional_tags("sage: #optional -- mypkg") set(['mypkg']) sage: parse_optional_tags("sage: print(1) # parentheses are optional here") set([]) sage: parse_optional_tags("sage: print(1) # optional") set(['']) sage: sorted(list(parse_optional_tags("sage: #optional -- foo bar, baz"))) ['bar', 'foo'] sage: sorted(list(parse_optional_tags(" sage: factor(10^(10^10) + 1) # LoNg TiME, NoT TeSTED; OptioNAL -- P4cka9e"))) ['long time', 'not tested', 'p4cka9e'] sage: parse_optional_tags(" sage: raise RuntimeError # known bug") set(['known bug']) sage: sorted(list(parse_optional_tags(" sage: determine_meaning_of_life() # long time, not implemented"))) ['long time', 'not implemented'] We don't parse inside strings:: sage: parse_optional_tags(" sage: print ' # long time'") set([]) sage: parse_optional_tags(" sage: print ' # long time' # not tested") set(['not tested']) """ safe, literals = strip_string_literals(string) first_line = safe.split('\n', 1)[0] if '#' not in first_line: return set() comment = first_line[first_line.find('#')+1:] # strip_string_literals replaces comments comment = "#" + (literals[comment]).lower() tags = [] for m in optional_regex.finditer(comment): if m.group(1): tags.append(m.group(1)) else: tags.extend(m.group(3).split() or [""]) return set(tags)
def parse_optional_tags(string): """ Returns a set consisting of the optional tags from the following set that occur in a comment on the first line of the input string. - 'long time' - 'not implemented' - 'not tested' - 'known bug' - 'optional: PKG_NAME' -- the set will just contain 'PKG_NAME' EXAMPLES:: sage: from sage.doctest.parsing import parse_optional_tags sage: parse_optional_tags("sage: magma('2 + 2')# optional: magma") set(['magma']) sage: parse_optional_tags("sage: #optional -- mypkg") set(['mypkg']) sage: parse_optional_tags("sage: print(1) # parentheses are optional here") set([]) sage: parse_optional_tags("sage: print(1) # optional") set(['']) sage: sorted(list(parse_optional_tags("sage: #optional -- foo bar, baz"))) ['bar', 'foo'] sage: sorted(list(parse_optional_tags(" sage: factor(10^(10^10) + 1) # LoNg TiME, NoT TeSTED; OptioNAL -- P4cka9e"))) ['long time', 'not tested', 'p4cka9e'] sage: parse_optional_tags(" sage: raise RuntimeError # known bug") set(['known bug']) sage: sorted(list(parse_optional_tags(" sage: determine_meaning_of_life() # long time, not implemented"))) ['long time', 'not implemented'] We don't parse inside strings:: sage: parse_optional_tags(" sage: print ' # long time'") set([]) sage: parse_optional_tags(" sage: print ' # long time' # not tested") set(['not tested']) """ safe, literals = strip_string_literals(string) first_line = safe.split('\n', 1)[0] if '#' not in first_line: return set() comment = first_line[first_line.find('#') + 1:] # strip_string_literals replaces comments comment = "#" + (literals[comment]).lower() tags = [] for m in optional_regex.finditer(comment): if m.group(1): tags.append(m.group(1)) else: tags.extend(m.group(3).split() or [""]) return set(tags)
def t(self, before, expected): actual, literals = strip_string_literals(before, prefix="_L") self.assertEqual(expected, actual) for key, value in literals.items(): actual = actual.replace(key, value) self.assertEqual(before, actual)
def cython_inline(code, get_type=unsafe_type, lib_dir=os.path.join(get_cython_cache_dir(), 'inline'), cython_include_dirs=['.'], force=False, quiet=False, locals=None, globals=None, **kwds): if get_type is None: get_type = lambda x: 'object' code = to_unicode(code) orig_code = code code, literals = strip_string_literals(code) code = strip_common_indent(code) ctx = Context(cython_include_dirs, default_options) if locals is None: locals = inspect.currentframe().f_back.f_back.f_locals if globals is None: globals = inspect.currentframe().f_back.f_back.f_globals try: for symbol in unbound_symbols(code): if symbol in kwds: continue elif symbol in locals: kwds[symbol] = locals[symbol] elif symbol in globals: kwds[symbol] = globals[symbol] else: print("Couldn't find ", symbol) except AssertionError: if not quiet: # Parsing from strings not fully supported (e.g. cimports). print("Could not parse code as a string (to extract unbound symbols).") cimports = [] for name, arg in kwds.items(): if arg is cython_module: cimports.append('\ncimport cython as %s' % name) del kwds[name] arg_names = kwds.keys() arg_names.sort() arg_sigs = tuple([(get_type(kwds[arg], ctx), arg) for arg in arg_names]) key = orig_code, arg_sigs, sys.version_info, sys.executable, Cython.__version__ module_name = "_cython_inline_" + hashlib.md5(str(key).encode('utf-8')).hexdigest() so_ext = [ ext for ext,_,mod_type in imp.get_suffixes() if mod_type == imp.C_EXTENSION ][0] module_path = os.path.join(lib_dir, module_name+so_ext) if not os.path.exists(lib_dir): os.makedirs(lib_dir) if force or not os.path.isfile(module_path): cflags = [] c_include_dirs = [] qualified = re.compile(r'([.\w]+)[.]') for type, _ in arg_sigs: m = qualified.match(type) if m: cimports.append('\ncimport %s' % m.groups()[0]) # one special case if m.groups()[0] == 'numpy': import numpy c_include_dirs.append(numpy.get_include()) # cflags.append('-Wno-unused') module_body, func_body = extract_func_code(code) params = ', '.join(['%s %s' % a for a in arg_sigs]) module_code = """ %(module_body)s %(cimports)s def __invoke(%(params)s): %(func_body)s """ % {'cimports': '\n'.join(cimports), 'module_body': module_body, 'params': params, 'func_body': func_body } for key, value in literals.items(): module_code = module_code.replace(key, value) pyx_file = os.path.join(lib_dir, module_name + '.pyx') fh = open(pyx_file, 'w') try: fh.write(module_code) finally: fh.close() extension = Extension( name = module_name, sources = [pyx_file], include_dirs = c_include_dirs, extra_compile_args = cflags) dist = Distribution() # Ensure the build respects distutils configuration by parsing # the configuration files config_files = dist.find_config_files() dist.parse_config_files(config_files) build_extension = build_ext(dist) build_extension.finalize_options() build_extension.extensions = cythonize([extension], ctx=ctx, quiet=quiet) build_extension.build_temp = os.path.dirname(pyx_file) build_extension.build_lib = lib_dir build_extension.run() _code_cache[key] = module_name module = imp.load_dynamic(module_name, module_path) arg_list = [kwds[arg] for arg in arg_names] return module.__invoke(*arg_list)
def t(self, before, expected): actual, literals = strip_string_literals(before, prefix="_L") self.assertEquals(expected, actual) for key, value in literals.items(): actual = actual.replace(key, value) self.assertEquals(before, actual)
def cython_inline(code, types='aggressive', lib_dir=os.path.expanduser('~/.cython/inline'), include_dirs=['.'], locals=None, globals=None, **kwds): code, literals = strip_string_literals(code) code = strip_common_indent(code) ctx = Context(include_dirs, default_options) if locals is None: locals = inspect.currentframe().f_back.f_back.f_locals if globals is None: globals = inspect.currentframe().f_back.f_back.f_globals try: for symbol in unbound_symbols(code): if symbol in kwds: continue elif symbol in locals: kwds[symbol] = locals[symbol] elif symbol in globals: kwds[symbol] = globals[symbol] else: print "Couldn't find ", symbol except AssertionError: # Parsing from strings not fully supported (e.g. cimports). print "Could not parse code as a string (to extract unbound symbols)." arg_names = kwds.keys() arg_names.sort() arg_sigs = tuple([(get_type(kwds[arg], ctx), arg) for arg in arg_names]) key = code, arg_sigs, sys.version_info, sys.executable, Cython.__version__ module_name = "_cython_inline_" + hashlib.md5(str(key)).hexdigest() # # TODO: Does this cover all the platforms? # if (not os.path.exists(os.path.join(lib_dir, module_name + ".so")) and # not os.path.exists(os.path.join(lib_dir, module_name + ".dll"))): try: if not os.path.exists(lib_dir): os.makedirs(lib_dir) if lib_dir not in sys.path: sys.path.append(lib_dir) __import__(module_name) except ImportError: c_include_dirs = [] cimports = [] qualified = re.compile(r'([.\w]+)[.]') for type, _ in arg_sigs: m = qualified.match(type) if m: cimports.append('\ncimport %s' % m.groups()[0]) # one special case if m.groups()[0] == 'numpy': import numpy c_include_dirs.append(numpy.get_include()) module_body, func_body = extract_func_code(code) params = ', '.join(['%s %s' % a for a in arg_sigs]) module_code = """ %(module_body)s %(cimports)s def __invoke(%(params)s): %(func_body)s """ % {'cimports': '\n'.join(cimports), 'module_body': module_body, 'params': params, 'func_body': func_body } for key, value in literals.items(): module_code = module_code.replace(key, value) pyx_file = os.path.join(lib_dir, module_name + '.pyx') open(pyx_file, 'w').write(module_code) extension = Extension( name = module_name, sources = [pyx_file], include_dirs = c_include_dirs) build_extension = build_ext(Distribution()) build_extension.finalize_options() build_extension.extensions = cythonize([extension]) build_extension.build_temp = os.path.dirname(pyx_file) build_extension.build_lib = lib_dir build_extension.run() _code_cache[key] = module_name arg_list = [kwds[arg] for arg in arg_names] return __import__(module_name).__invoke(*arg_list)
def cython_inline(code, get_type=unsafe_type, lib_dir=os.path.join(get_cython_cache_dir(), 'inline'), cython_include_dirs=None, force=False, quiet=False, local_sym=None, global_sym=None, **kwargs): if get_type is None: get_type = lambda x: 'object' ctx = _create_context(tuple(cython_include_dirs) ) if cython_include_dirs else _inline_default_context # Fast path if this has been called in this session. _unbound_symbols = _inline_cache.get(code) if _unbound_symbols is not None: _populate_unbound(kwargs, _unbound_symbols, local_sym, global_sym) args = sorted(kwargs.items()) arg_sigs = tuple([(get_type(value, ctx), arg) for arg, value in args]) invoke = _inline_cache.get((code, arg_sigs)) if invoke is not None: arg_list = [arg[1] for arg in args] return invoke(*arg_list) orig_code = code code, literals = strip_string_literals(code) if local_sym is None: local_sym = inspect.currentframe().f_back.f_back.f_locals if global_sym is None: global_sym = inspect.currentframe().f_back.f_back.f_globals try: _inline_cache[orig_code] = _unbound_symbols = unbound_symbols(code) _populate_unbound(kwargs, _unbound_symbols, local_sym, global_sym) except AssertionError: if not quiet: # Parsing from strings not fully supported (e.g. cimports). print( "Could not parse code as a string (to extract unbound symbols)." ) cimports = [] for name, arg in list(kwargs.items()): if arg is cython: cimports.append('\ncimport cython as %s' % name) del kwargs[name] arg_names = sorted(kwargs) arg_sigs = tuple([(get_type(kwargs[arg], ctx), arg) for arg in arg_names]) key = orig_code, arg_sigs, sys.version_info, sys.executable, Cython.__version__ module_name = "_cython_inline_" + hashlib.md5( str(key).encode('utf-8')).hexdigest() if module_name in sys.modules: _module = sys.modules[module_name] else: build_extension = None global _so_ext if _so_ext is None: # Figure out and cache current extension suffix build_extension = _get_build_extension() _so_ext = build_extension.get_ext_filename('') module_path = os.path.join(lib_dir, module_name + _so_ext) if not os.path.exists(lib_dir): os.makedirs(lib_dir) if force or not os.path.isfile(module_path): cflags = [] c_include_dirs = [] qualified = re.compile(r'([.\w]+)[.]') for type, _ in arg_sigs: m = qualified.match(type) if m: cimports.append('\ncimport %s' % m.groups()[0]) # one special case if m.groups()[0] == 'numpy': import numpy c_include_dirs.append(numpy.get_include()) # cflags.append('-Wno-unused') module_body, func_body = extract_func_code(code) params = ', '.join(['%s %s' % a for a in arg_sigs]) module_code = ''' %(module_body)s %(cimports)s def __invoke(%(params)s): %(func_body)s return locals() ''' % { 'cimports': '\n'.join(cimports), 'module_body': module_body, 'params': params, 'func_body': func_body } for key, value in literals.items(): module_code = module_code.replace(key, value) pyx_file = os.path.join(lib_dir, module_name + '.pyx') fh = open(pyx_file, 'w') try: fh.write(module_code) finally: fh.close() extension = Extension(name=module_name, sources=[pyx_file], include_dirs=c_include_dirs, extra_compile_args=cflags) if build_extension is None: build_extension = _get_build_extension() build_extension.extensions = cythonize( [extension], include_path=cython_include_dirs or ['.'], quiet=quiet) build_extension.build_temp = os.path.dirname(pyx_file) build_extension.build_lib = lib_dir build_extension.run() _module = imp.load_dynamic(module_name, module_path) _inline_cache[orig_code, arg_sigs] = _module.__invoke arg_list = [kwargs[arg] for arg in arg_names] return _module.__invoke(*arg_list)
def cython_inline( code, get_type=unsafe_type, lib_dir=os.path.join(get_cython_cache_dir(), "inline"), cython_include_dirs=None, force=False, quiet=False, locals=None, globals=None, **kwds ): if cython_include_dirs is None: cython_include_dirs = ["."] if get_type is None: get_type = lambda x: "object" code = to_unicode(code) orig_code = code code, literals = strip_string_literals(code) code = strip_common_indent(code) ctx = _create_context(tuple(cython_include_dirs)) if locals is None: locals = inspect.currentframe().f_back.f_back.f_locals if globals is None: globals = inspect.currentframe().f_back.f_back.f_globals try: for symbol in unbound_symbols(code): if symbol in kwds: continue elif symbol in locals: kwds[symbol] = locals[symbol] elif symbol in globals: kwds[symbol] = globals[symbol] else: print("Couldn't find ", symbol) except AssertionError: if not quiet: # Parsing from strings not fully supported (e.g. cimports). print("Could not parse code as a string (to extract unbound symbols).") cimports = [] for name, arg in kwds.items(): if arg is cython_module: cimports.append("\ncimport cython as %s" % name) del kwds[name] arg_names = kwds.keys() arg_names.sort() arg_sigs = tuple([(get_type(kwds[arg], ctx), arg) for arg in arg_names]) key = orig_code, arg_sigs, sys.version_info, sys.executable, Cython.__version__ module_name = "_cython_inline_" + hashlib.md5(str(key).encode("utf-8")).hexdigest() if module_name in sys.modules: module = sys.modules[module_name] else: build_extension = None if cython_inline.so_ext is None: # Figure out and cache current extension suffix build_extension = _get_build_extension() cython_inline.so_ext = build_extension.get_ext_filename("") module_path = os.path.join(lib_dir, module_name + cython_inline.so_ext) if not os.path.exists(lib_dir): os.makedirs(lib_dir) if force or not os.path.isfile(module_path): cflags = [] c_include_dirs = [] qualified = re.compile(r"([.\w]+)[.]") for type, _ in arg_sigs: m = qualified.match(type) if m: cimports.append("\ncimport %s" % m.groups()[0]) # one special case if m.groups()[0] == "numpy": import numpy c_include_dirs.append(numpy.get_include()) # cflags.append('-Wno-unused') module_body, func_body = extract_func_code(code) params = ", ".join(["%s %s" % a for a in arg_sigs]) module_code = """ %(module_body)s %(cimports)s def __invoke(%(params)s): %(func_body)s """ % { "cimports": "\n".join(cimports), "module_body": module_body, "params": params, "func_body": func_body, } for key, value in literals.items(): module_code = module_code.replace(key, value) pyx_file = os.path.join(lib_dir, module_name + ".pyx") fh = open(pyx_file, "w") try: fh.write(module_code) finally: fh.close() extension = Extension( name=module_name, sources=[pyx_file], include_dirs=c_include_dirs, extra_compile_args=cflags ) if build_extension is None: build_extension = _get_build_extension() build_extension.extensions = cythonize([extension], include_path=cython_include_dirs, quiet=quiet) build_extension.build_temp = os.path.dirname(pyx_file) build_extension.build_lib = lib_dir build_extension.run() module = imp.load_dynamic(module_name, module_path) arg_list = [kwds[arg] for arg in arg_names] return module.__invoke(*arg_list)