def generate_pyi_from_file(file: Path) -> bool:
"""Generate a .pyi stubfile from a single .py module using mypy/stubgen"""
sg_opt = stubgen.Options(
pyversion=(3, 6),
no_import=False,
include_private=True,
doc_dir="",
search_path=[],
interpreter=sys.executable,
parse_only=False,
ignore_errors=True,
modules=[],
packages=[],
files=[],
output_dir="",
verbose=True,
quiet=False,
export_less=False,
)
# Deal with generator passed in
if not isinstance(file, Path):
raise TypeError
sg_opt.files = [str(file)]
sg_opt.output_dir = str(file.parent)
try:
print(f"Calling stubgen on {str(file)}")
# WORKAROUND: Stubgen.generate_stubs does not provide a way to return the errors
# such as `cannot perform relative import`
# UPSTREAM: robust.py needs this to do a relative import
if file.name == "robust.py" and file.parent.name == "umqtt":
with open(file.with_name("__init__.py"), "a") as init:
init.write("# force __init__.py")
stubgen.generate_stubs(sg_opt)
return True
except (Exception, CompileError, SystemExit) as e:
print(e)
return False
def generate_typed_stubs(modpath):
"""
Attempt to use google-style docstrings, xdoctest, and mypy to generate
typed stub files.
Does not overwrite anything by itself.
Args:
modpath (PathLike): path to the module to generate types for
Returns:
Dict[PathLike, str]:
A dictionary mapping the path of each file to write to the text to
be written.
Notes:
FIXME: This currently requires my hacked version of mypy
Example:
>>> # xdoctest: +SKIP
>>> # xdoctest: +REQUIRES(module:mypy)
>>> # xdoctest: +REQUIRES(--hacked)
>>> from xdev.cli.docstr_stubgen import * # NOQA
>>> import xdev
>>> import ubelt as ub
>>> from xdev.cli import docstr_stubgen
>>> modpath = ub.Path(docstr_stubgen.__file__)
>>> generated = generate_typed_stubs(modpath)
>>> text = generated[ub.peek(generated.keys())]
>>> assert 'PathLike' in text
>>> assert 'Dict' in text
>>> print(text)
Ignore:
pyfile mypy.stubgen
# Delete compiled verisons so we can hack it
# ls $VIRTUAL_ENV/lib/*/site-packages/mypy/*.so
# rm $VIRTUAL_ENV/lib/*/site-packages/mypy/*.so
# rm ~/.pyenv/versions/3.8.6/envs/pyenv3.8.6/lib/python3.8/site-packages/mypy/*.cpython-38-x86_64-linux-gnu.so
# This works I think?
if [[ ! -e "$HOME/code/mypy" ]]; then
git clone https://github.com/python/mypy.git $HOME/code/mypy
fi
(cd $HOME/code/mypy && git pull)
pip install -e $HOME/code/mypy
pip install MonkeyType
monkeytype run run_tests.py
monkeytype stub ubelt.util_dict
from typing import TypeVar
from mypy.applytype import get_target_type
z = TypeVar('Iterable')
get_target_type(z)
from mypy.expandtype import expand_type
expand_type(z, env={})
from mypy.types import get_proper_type
get_proper_type(z)
get_proper_type(dict)
import typing
get_proper_type(typing.Iterable)
from mypy.types import deserialize_type, UnboundType
import mypy.types as mypy_types
z = UnboundType('Iterable')
get_proper_type(dict)
from mypy.fastparse import parse_type_string
parse_type_string('dict', 'dict', 0, 0)
z = parse_type_string('typing.Iterator', 'Any', 0, 0)
get_proper_type(z)
"""
# import pathlib
# import ubelt as ub
import os
from mypy import stubgen
from mypy import defaults
from xdoctest import static_analysis
from os.path import join
import ubelt as ub
# modname = 'scriptconfig'
# module = ub.import_module_from_name(modname)
# modpath = ub.Path(module.__file__).parent
# for p in pathlib.Path(modpath).glob('*.pyi'):
# p.unlink()
modpath = ub.Path(modpath)
files = list(
static_analysis.package_modpaths(modpath,
recursive=True,
with_libs=1,
with_pkg=0))
# files = [f for f in files if 'deprecated' not in f]
# files = [join(ubelt_dpath, 'util_dict.py')]
if modpath.is_file():
output_dir = modpath.parent.parent
else:
output_dir = modpath.parent
options = stubgen.Options(pyversion=defaults.PYTHON3_VERSION,
no_import=True,
doc_dir='',
search_path=[],
interpreter=sys.executable,
ignore_errors=False,
parse_only=True,
include_private=False,
output_dir=output_dir,
modules=[],
packages=[],
files=files,
verbose=False,
quiet=False,
export_less=True)
# generate_stubs(options)
mypy_opts = stubgen.mypy_options(options)
py_modules, c_modules = stubgen.collect_build_targets(options, mypy_opts)
# Collect info from docs (if given):
sigs = class_sigs = None # type: Optional[Dict[str, str]]
if options.doc_dir:
sigs, class_sigs = stubgen.collect_docs_signatures(options.doc_dir)
# Use parsed sources to generate stubs for Python modules.
stubgen.generate_asts_for_modules(py_modules, options.parse_only,
mypy_opts, options.verbose)
generated = {}
for mod in py_modules:
assert mod.path is not None, "Not found module was not skipped"
target = mod.module.replace('.', '/')
if os.path.basename(mod.path) == '__init__.py':
target += '/__init__.pyi'
else:
target += '.pyi'
target = join(options.output_dir, target)
files.append(target)
with stubgen.generate_guarded(mod.module, target,
options.ignore_errors, options.verbose):
stubgen.generate_stub_from_ast(mod, target, options.parse_only,
options.pyversion,
options.include_private,
options.export_less)
gen = ExtendedStubGenerator(
mod.runtime_all,
pyversion=options.pyversion,
include_private=options.include_private,
analyzed=not options.parse_only,
export_less=options.export_less)
assert mod.ast is not None, "This function must be used only with analyzed modules"
mod.ast.accept(gen)
# print('gen.import_tracker.required_names = {!r}'.format(gen.import_tracker.required_names))
# print(gen.import_tracker.import_lines())
# print('mod.path = {!r}'.format(mod.path))
known_one_letter_types = {
# 'T', 'K', 'A', 'B', 'C', 'V',
'DT',
'KT',
'VT',
'T'
}
for type_var_name in set(gen.import_tracker.required_names) & set(
known_one_letter_types):
gen.add_typing_import('TypeVar')
# gen.add_import_line('from typing import {}\n'.format('TypeVar'))
gen._output = [
'{} = TypeVar("{}")\n'.format(type_var_name, type_var_name)
] + gen._output
custom_types = {'Hasher'}
for type_var_name in set(
gen.import_tracker.required_names) & set(custom_types):
gen.add_typing_import('TypeVar')
# gen.add_import_line('from typing import {}\n'.format('TypeVar'))
gen._output = [
'{} = TypeVar("{}")\n'.format(type_var_name, type_var_name)
] + gen._output
# Hack for specific module
# if mod.path.endswith('util_path.py'):
# gen.add_typing_import('TypeVar')
# # hack for variable inheritence
# gen._output = ['import pathlib\nimport os\n', "_PathBase = pathlib.WindowsPath if os.name == 'nt' else pathlib.PosixPath\n"] + gen._output
text = ''.join(gen.output())
text = postprocess_hacks(text, mod)
# Write output to file.
# subdir = ub.Path(target).parent
# if subdir and not os.path.isdir(subdir):
# os.makedirs(subdir)
generated[target] = text
# with open(target, 'w') as file:
# file.write(text)
return generated
def generate_typed_stubs():
"""
Attempt to use google-style docstrings, xdoctest, and mypy to generate
typed stub files.
pyfile mypy.stubgen
# Delete compiled verisons so we can hack it
rm ~/.pyenv/versions/3.8.6/envs/pyenv3.8.6/lib/python3.8/site-packages/mypy/*.cpython-38-x86_64-linux-gnu.so
git clone https://github.com/python/mypy.git
cd mypy
pip install -e .
pip install MonkeyType
monkeytype run run_tests.py
monkeytype stub ubelt.util_dict
from typing import TypeVar
from mypy.applytype import get_target_type
z = TypeVar('Iterable')
get_target_type(z)
from mypy.expandtype import expand_type
expand_type(z, env={})
from mypy.types import get_proper_type
get_proper_type(z)
get_proper_type(dict)
import typing
get_proper_type(typing.Iterable)
from mypy.types import deserialize_type, UnboundType
import mypy.types as mypy_types
z = UnboundType('Iterable')
get_proper_type(dict)
from mypy.fastparse import parse_type_string
parse_type_string('dict', 'dict', 0, 0)
z = parse_type_string('typing.Iterator', 'Any', 0, 0)
get_proper_type(z)
"""
import pathlib
import ubelt
import os
import autoflake
import yapf
from mypy import stubgen
from mypy import defaults
from xdoctest import static_analysis
from os.path import dirname, join
ubelt_dpath = dirname(ubelt.__file__)
for p in pathlib.Path(ubelt_dpath).glob('*.pyi'):
p.unlink()
files = list(
static_analysis.package_modpaths(ubelt_dpath,
recursive=True,
with_libs=1,
with_pkg=0))
files = [f for f in files if 'deprecated' not in f]
# files = [join(ubelt_dpath, 'util_dict.py')]
options = stubgen.Options(pyversion=defaults.PYTHON3_VERSION,
no_import=True,
doc_dir='',
search_path=[],
interpreter=sys.executable,
ignore_errors=False,
parse_only=True,
include_private=False,
output_dir=dirname(ubelt_dpath),
modules=[],
packages=[],
files=files,
verbose=False,
quiet=False,
export_less=True)
# generate_stubs(options)
mypy_opts = stubgen.mypy_options(options)
py_modules, c_modules = stubgen.collect_build_targets(options, mypy_opts)
# Collect info from docs (if given):
sigs = class_sigs = None # type: Optional[Dict[str, str]]
if options.doc_dir:
sigs, class_sigs = stubgen.collect_docs_signatures(options.doc_dir)
# Use parsed sources to generate stubs for Python modules.
stubgen.generate_asts_for_modules(py_modules, options.parse_only,
mypy_opts, options.verbose)
for mod in py_modules:
assert mod.path is not None, "Not found module was not skipped"
target = mod.module.replace('.', '/')
if os.path.basename(mod.path) == '__init__.py':
target += '/__init__.pyi'
else:
target += '.pyi'
target = join(options.output_dir, target)
files.append(target)
with stubgen.generate_guarded(mod.module, target,
options.ignore_errors, options.verbose):
stubgen.generate_stub_from_ast(mod, target, options.parse_only,
options.pyversion,
options.include_private,
options.export_less)
gen = ExtendedStubGenerator(
mod.runtime_all,
pyversion=options.pyversion,
include_private=options.include_private,
analyzed=not options.parse_only,
export_less=options.export_less)
assert mod.ast is not None, "This function must be used only with analyzed modules"
mod.ast.accept(gen)
# print('gen.import_tracker.required_names = {!r}'.format(gen.import_tracker.required_names))
# print(gen.import_tracker.import_lines())
print('mod.path = {!r}'.format(mod.path))
known_one_letter_types = {
# 'T', 'K', 'A', 'B', 'C', 'V',
'DT',
'KT',
'VT',
'T'
}
for type_var_name in set(gen.import_tracker.required_names) & set(
known_one_letter_types):
gen.add_typing_import('TypeVar')
# gen.add_import_line('from typing import {}\n'.format('TypeVar'))
gen._output = [
'{} = TypeVar("{}")\n'.format(type_var_name, type_var_name)
] + gen._output
custom_types = {'Hasher'}
for type_var_name in set(
gen.import_tracker.required_names) & set(custom_types):
gen.add_typing_import('TypeVar')
# gen.add_import_line('from typing import {}\n'.format('TypeVar'))
gen._output = [
'{} = TypeVar("{}")\n'.format(type_var_name, type_var_name)
] + gen._output
# Hack for specific module
# if mod.path.endswith('util_path.py'):
# gen.add_typing_import('TypeVar')
# # hack for variable inheritence
# gen._output = ['import pathlib\nimport os\n', "_PathBase = pathlib.WindowsPath if os.name == 'nt' else pathlib.PosixPath\n"] + gen._output
text = ''.join(gen.output())
# Hack to remove lines caused by Py2 compat
text = text.replace('Generator = object\n', '')
text = text.replace('select = NotImplemented\n', '')
text = text.replace('iteritems: Any\n', '')
text = text.replace('text_type = str\n', '')
text = text.replace('text_type: Any\n', '')
text = text.replace('string_types: Any\n', '')
text = text.replace('PY2: Any\n', '')
text = text.replace('__win32_can_symlink__: Any\n', '')
# text = text.replace('odict = OrderedDict', '')
# text = text.replace('ddict = defaultdict', '')
if mod.path.endswith('util_path.py'):
# hack for forward reference
text = text.replace(' -> Path:', " -> 'Path':")
text = text.replace('class Path(_PathBase)', "class Path")
# Format the PYI file nicely
text = autoflake.fix_code(text,
remove_unused_variables=True,
remove_all_unused_imports=True)
# import autopep8
# text = autopep8.fix_code(text, options={
# 'aggressive': 0,
# 'experimental': 0,
# })
style = yapf.yapf_api.style.CreatePEP8Style()
text, _ = yapf.yapf_api.FormatCode(text,
filename='<stdin>',
style_config=style,
lines=None,
verify=False)
# print(text)
# Write output to file.
subdir = dirname(target)
if subdir and not os.path.isdir(subdir):
os.makedirs(subdir)
with open(target, 'w') as file:
file.write(text)