def generate_stubs() -> None:
"""Main entry point for the program."""
options = Options(
pyversion=sys.version_info[:2],
no_import=True,
doc_dir="",
search_path=[],
interpreter=sys.executable,
parse_only=False,
ignore_errors=False,
include_private=False,
output_dir="",
modules=[],
packages=[],
files=["devolo_plc_api/device_api/deviceapi.py", "devolo_plc_api/plcnet_api/plcnetapi.py"],
verbose=False,
quiet=True,
export_less=True,
)
mypy_opts = mypy_options(options)
py_modules, _ = collect_build_targets(options, mypy_opts)
generate_asts_for_modules(py_modules, options.parse_only, mypy_opts, options.verbose)
files = []
for mod in py_modules:
target = mod.module.replace(".", "/")
target += ".pyi"
target = os.path.join(options.output_dir, target)
files.append(target)
with generate_guarded(mod.module, target, options.ignore_errors, options.verbose):
generate_stub_from_ast(
mod, target, options.parse_only, options.pyversion, options.include_private, options.export_less
)
def test_packages_found(self) -> None:
current = os.getcwd()
with tempfile.TemporaryDirectory() as tmp:
try:
os.chdir(tmp)
os.mkdir('pack')
self.make_file('pack',
'__init__.py',
content='from . import a, b')
self.make_file('pack', 'a.py')
self.make_file('pack', 'b.py')
opts = parse_options(['-p', 'pack'])
py_mods, c_mods = collect_build_targets(
opts, mypy_options(opts))
assert_equal(c_mods, [])
files = {
os.path.relpath(mod.path or 'FAIL')
for mod in py_mods
}
assert_equal(
files, {
os.path.join('pack', '__init__.py'),
os.path.join('pack', 'a.py'),
os.path.join('pack', 'b.py')
})
finally:
os.chdir(current)
def test_module_not_found(self) -> None:
current = os.getcwd()
captured_output = io.StringIO()
sys.stdout = captured_output
with tempfile.TemporaryDirectory() as tmp:
try:
os.chdir(tmp)
self.make_file(tmp, 'mymodule.py', content='import a')
opts = parse_options(['-m', 'mymodule'])
py_mods, c_mods = collect_build_targets(opts, mypy_options(opts))
assert captured_output.getvalue() == ''
finally:
sys.stdout = sys.__stdout__
os.chdir(current)
def test_module_not_found(self) -> None:
current = os.getcwd()
captured_output = io.StringIO()
sys.stdout = captured_output
with tempfile.TemporaryDirectory() as tmp:
try:
os.chdir(tmp)
self.make_file(tmp, 'mymodule.py', content='import a')
opts = parse_options(['-m', 'mymodule'])
py_mods, c_mods = collect_build_targets(opts, mypy_options(opts))
self.assertRegex(captured_output.getvalue(), "No module named 'a'")
finally:
sys.stdout = sys.__stdout__
os.chdir(current)
def test_packages_found(self) -> None:
current = os.getcwd()
with tempfile.TemporaryDirectory() as tmp:
try:
os.chdir(tmp)
os.mkdir('pack')
self.make_file('pack', '__init__.py', content='from . import a, b')
self.make_file('pack', 'a.py')
self.make_file('pack', 'b.py')
opts = parse_options(['-p', 'pack'])
py_mods, c_mods = collect_build_targets(opts, mypy_options(opts))
assert_equal(c_mods, [])
files = {os.path.relpath(mod.path or 'FAIL') for mod in py_mods}
assert_equal(files, {os.path.join('pack', '__init__.py'),
os.path.join('pack', 'a.py'),
os.path.join('pack', 'b.py')})
finally:
os.chdir(current)
def test_files_found(self) -> None:
current = os.getcwd()
with tempfile.TemporaryDirectory() as tmp:
try:
os.chdir(tmp)
os.mkdir('subdir')
self.make_file('subdir', 'a.py')
self.make_file('subdir', 'b.py')
os.mkdir(os.path.join('subdir', 'pack'))
self.make_file('subdir', 'pack', '__init__.py')
opts = parse_options(['subdir'])
py_mods, c_mods = collect_build_targets(opts, mypy_options(opts))
assert_equal(c_mods, [])
files = {mod.path for mod in py_mods}
assert_equal(files, {os.path.join('subdir', 'pack', '__init__.py'),
os.path.join('subdir', 'a.py'),
os.path.join('subdir', 'b.py')})
finally:
os.chdir(current)
def test_files_found(self) -> None:
current = os.getcwd()
with tempfile.TemporaryDirectory() as tmp:
try:
os.chdir(tmp)
os.mkdir('subdir')
self.make_file('subdir', 'a.py')
self.make_file('subdir', 'b.py')
os.mkdir(os.path.join('subdir', 'pack'))
self.make_file('subdir', 'pack', '__init__.py')
opts = parse_options(['subdir'])
py_mods, c_mods = collect_build_targets(opts, mypy_options(opts))
assert_equal(c_mods, [])
files = {mod.path for mod in py_mods}
assert_equal(files, {os.path.join('subdir', 'pack', '__init__.py'),
os.path.join('subdir', 'a.py'),
os.path.join('subdir', 'b.py')})
finally:
os.chdir(current)
def generate_typed_stubs(modpath):
"""
Attempt to use google-style docstrings, xdoctest, and mypy to generate
typed stub files.
Does not overwrite anything by itself.
Args:
modpath (PathLike): path to the module to generate types for
Returns:
Dict[PathLike, str]:
A dictionary mapping the path of each file to write to the text to
be written.
Notes:
FIXME: This currently requires my hacked version of mypy
Example:
>>> # xdoctest: +SKIP
>>> # xdoctest: +REQUIRES(module:mypy)
>>> # xdoctest: +REQUIRES(--hacked)
>>> from xdev.cli.docstr_stubgen import * # NOQA
>>> import xdev
>>> import ubelt as ub
>>> from xdev.cli import docstr_stubgen
>>> modpath = ub.Path(docstr_stubgen.__file__)
>>> generated = generate_typed_stubs(modpath)
>>> text = generated[ub.peek(generated.keys())]
>>> assert 'PathLike' in text
>>> assert 'Dict' in text
>>> print(text)
Ignore:
pyfile mypy.stubgen
# Delete compiled verisons so we can hack it
# ls $VIRTUAL_ENV/lib/*/site-packages/mypy/*.so
# rm $VIRTUAL_ENV/lib/*/site-packages/mypy/*.so
# rm ~/.pyenv/versions/3.8.6/envs/pyenv3.8.6/lib/python3.8/site-packages/mypy/*.cpython-38-x86_64-linux-gnu.so
# This works I think?
if [[ ! -e "$HOME/code/mypy" ]]; then
git clone https://github.com/python/mypy.git $HOME/code/mypy
fi
(cd $HOME/code/mypy && git pull)
pip install -e $HOME/code/mypy
pip install MonkeyType
monkeytype run run_tests.py
monkeytype stub ubelt.util_dict
from typing import TypeVar
from mypy.applytype import get_target_type
z = TypeVar('Iterable')
get_target_type(z)
from mypy.expandtype import expand_type
expand_type(z, env={})
from mypy.types import get_proper_type
get_proper_type(z)
get_proper_type(dict)
import typing
get_proper_type(typing.Iterable)
from mypy.types import deserialize_type, UnboundType
import mypy.types as mypy_types
z = UnboundType('Iterable')
get_proper_type(dict)
from mypy.fastparse import parse_type_string
parse_type_string('dict', 'dict', 0, 0)
z = parse_type_string('typing.Iterator', 'Any', 0, 0)
get_proper_type(z)
"""
# import pathlib
# import ubelt as ub
import os
from mypy import stubgen
from mypy import defaults
from xdoctest import static_analysis
from os.path import join
import ubelt as ub
# modname = 'scriptconfig'
# module = ub.import_module_from_name(modname)
# modpath = ub.Path(module.__file__).parent
# for p in pathlib.Path(modpath).glob('*.pyi'):
# p.unlink()
modpath = ub.Path(modpath)
files = list(
static_analysis.package_modpaths(modpath,
recursive=True,
with_libs=1,
with_pkg=0))
# files = [f for f in files if 'deprecated' not in f]
# files = [join(ubelt_dpath, 'util_dict.py')]
if modpath.is_file():
output_dir = modpath.parent.parent
else:
output_dir = modpath.parent
options = stubgen.Options(pyversion=defaults.PYTHON3_VERSION,
no_import=True,
doc_dir='',
search_path=[],
interpreter=sys.executable,
ignore_errors=False,
parse_only=True,
include_private=False,
output_dir=output_dir,
modules=[],
packages=[],
files=files,
verbose=False,
quiet=False,
export_less=True)
# generate_stubs(options)
mypy_opts = stubgen.mypy_options(options)
py_modules, c_modules = stubgen.collect_build_targets(options, mypy_opts)
# Collect info from docs (if given):
sigs = class_sigs = None # type: Optional[Dict[str, str]]
if options.doc_dir:
sigs, class_sigs = stubgen.collect_docs_signatures(options.doc_dir)
# Use parsed sources to generate stubs for Python modules.
stubgen.generate_asts_for_modules(py_modules, options.parse_only,
mypy_opts, options.verbose)
generated = {}
for mod in py_modules:
assert mod.path is not None, "Not found module was not skipped"
target = mod.module.replace('.', '/')
if os.path.basename(mod.path) == '__init__.py':
target += '/__init__.pyi'
else:
target += '.pyi'
target = join(options.output_dir, target)
files.append(target)
with stubgen.generate_guarded(mod.module, target,
options.ignore_errors, options.verbose):
stubgen.generate_stub_from_ast(mod, target, options.parse_only,
options.pyversion,
options.include_private,
options.export_less)
gen = ExtendedStubGenerator(
mod.runtime_all,
pyversion=options.pyversion,
include_private=options.include_private,
analyzed=not options.parse_only,
export_less=options.export_less)
assert mod.ast is not None, "This function must be used only with analyzed modules"
mod.ast.accept(gen)
# print('gen.import_tracker.required_names = {!r}'.format(gen.import_tracker.required_names))
# print(gen.import_tracker.import_lines())
# print('mod.path = {!r}'.format(mod.path))
known_one_letter_types = {
# 'T', 'K', 'A', 'B', 'C', 'V',
'DT',
'KT',
'VT',
'T'
}
for type_var_name in set(gen.import_tracker.required_names) & set(
known_one_letter_types):
gen.add_typing_import('TypeVar')
# gen.add_import_line('from typing import {}\n'.format('TypeVar'))
gen._output = [
'{} = TypeVar("{}")\n'.format(type_var_name, type_var_name)
] + gen._output
custom_types = {'Hasher'}
for type_var_name in set(
gen.import_tracker.required_names) & set(custom_types):
gen.add_typing_import('TypeVar')
# gen.add_import_line('from typing import {}\n'.format('TypeVar'))
gen._output = [
'{} = TypeVar("{}")\n'.format(type_var_name, type_var_name)
] + gen._output
# Hack for specific module
# if mod.path.endswith('util_path.py'):
# gen.add_typing_import('TypeVar')
# # hack for variable inheritence
# gen._output = ['import pathlib\nimport os\n', "_PathBase = pathlib.WindowsPath if os.name == 'nt' else pathlib.PosixPath\n"] + gen._output
text = ''.join(gen.output())
text = postprocess_hacks(text, mod)
# Write output to file.
# subdir = ub.Path(target).parent
# if subdir and not os.path.isdir(subdir):
# os.makedirs(subdir)
generated[target] = text
# with open(target, 'w') as file:
# file.write(text)
return generated
def generate_typed_stubs():
"""
Attempt to use google-style docstrings, xdoctest, and mypy to generate
typed stub files.
pyfile mypy.stubgen
# Delete compiled verisons so we can hack it
rm ~/.pyenv/versions/3.8.6/envs/pyenv3.8.6/lib/python3.8/site-packages/mypy/*.cpython-38-x86_64-linux-gnu.so
git clone https://github.com/python/mypy.git
cd mypy
pip install -e .
pip install MonkeyType
monkeytype run run_tests.py
monkeytype stub ubelt.util_dict
from typing import TypeVar
from mypy.applytype import get_target_type
z = TypeVar('Iterable')
get_target_type(z)
from mypy.expandtype import expand_type
expand_type(z, env={})
from mypy.types import get_proper_type
get_proper_type(z)
get_proper_type(dict)
import typing
get_proper_type(typing.Iterable)
from mypy.types import deserialize_type, UnboundType
import mypy.types as mypy_types
z = UnboundType('Iterable')
get_proper_type(dict)
from mypy.fastparse import parse_type_string
parse_type_string('dict', 'dict', 0, 0)
z = parse_type_string('typing.Iterator', 'Any', 0, 0)
get_proper_type(z)
"""
import pathlib
import ubelt
import os
import autoflake
import yapf
from mypy import stubgen
from mypy import defaults
from xdoctest import static_analysis
from os.path import dirname, join
ubelt_dpath = dirname(ubelt.__file__)
for p in pathlib.Path(ubelt_dpath).glob('*.pyi'):
p.unlink()
files = list(
static_analysis.package_modpaths(ubelt_dpath,
recursive=True,
with_libs=1,
with_pkg=0))
files = [f for f in files if 'deprecated' not in f]
# files = [join(ubelt_dpath, 'util_dict.py')]
options = stubgen.Options(pyversion=defaults.PYTHON3_VERSION,
no_import=True,
doc_dir='',
search_path=[],
interpreter=sys.executable,
ignore_errors=False,
parse_only=True,
include_private=False,
output_dir=dirname(ubelt_dpath),
modules=[],
packages=[],
files=files,
verbose=False,
quiet=False,
export_less=True)
# generate_stubs(options)
mypy_opts = stubgen.mypy_options(options)
py_modules, c_modules = stubgen.collect_build_targets(options, mypy_opts)
# Collect info from docs (if given):
sigs = class_sigs = None # type: Optional[Dict[str, str]]
if options.doc_dir:
sigs, class_sigs = stubgen.collect_docs_signatures(options.doc_dir)
# Use parsed sources to generate stubs for Python modules.
stubgen.generate_asts_for_modules(py_modules, options.parse_only,
mypy_opts, options.verbose)
for mod in py_modules:
assert mod.path is not None, "Not found module was not skipped"
target = mod.module.replace('.', '/')
if os.path.basename(mod.path) == '__init__.py':
target += '/__init__.pyi'
else:
target += '.pyi'
target = join(options.output_dir, target)
files.append(target)
with stubgen.generate_guarded(mod.module, target,
options.ignore_errors, options.verbose):
stubgen.generate_stub_from_ast(mod, target, options.parse_only,
options.pyversion,
options.include_private,
options.export_less)
gen = ExtendedStubGenerator(
mod.runtime_all,
pyversion=options.pyversion,
include_private=options.include_private,
analyzed=not options.parse_only,
export_less=options.export_less)
assert mod.ast is not None, "This function must be used only with analyzed modules"
mod.ast.accept(gen)
# print('gen.import_tracker.required_names = {!r}'.format(gen.import_tracker.required_names))
# print(gen.import_tracker.import_lines())
print('mod.path = {!r}'.format(mod.path))
known_one_letter_types = {
# 'T', 'K', 'A', 'B', 'C', 'V',
'DT',
'KT',
'VT',
'T'
}
for type_var_name in set(gen.import_tracker.required_names) & set(
known_one_letter_types):
gen.add_typing_import('TypeVar')
# gen.add_import_line('from typing import {}\n'.format('TypeVar'))
gen._output = [
'{} = TypeVar("{}")\n'.format(type_var_name, type_var_name)
] + gen._output
custom_types = {'Hasher'}
for type_var_name in set(
gen.import_tracker.required_names) & set(custom_types):
gen.add_typing_import('TypeVar')
# gen.add_import_line('from typing import {}\n'.format('TypeVar'))
gen._output = [
'{} = TypeVar("{}")\n'.format(type_var_name, type_var_name)
] + gen._output
# Hack for specific module
# if mod.path.endswith('util_path.py'):
# gen.add_typing_import('TypeVar')
# # hack for variable inheritence
# gen._output = ['import pathlib\nimport os\n', "_PathBase = pathlib.WindowsPath if os.name == 'nt' else pathlib.PosixPath\n"] + gen._output
text = ''.join(gen.output())
# Hack to remove lines caused by Py2 compat
text = text.replace('Generator = object\n', '')
text = text.replace('select = NotImplemented\n', '')
text = text.replace('iteritems: Any\n', '')
text = text.replace('text_type = str\n', '')
text = text.replace('text_type: Any\n', '')
text = text.replace('string_types: Any\n', '')
text = text.replace('PY2: Any\n', '')
text = text.replace('__win32_can_symlink__: Any\n', '')
# text = text.replace('odict = OrderedDict', '')
# text = text.replace('ddict = defaultdict', '')
if mod.path.endswith('util_path.py'):
# hack for forward reference
text = text.replace(' -> Path:', " -> 'Path':")
text = text.replace('class Path(_PathBase)', "class Path")
# Format the PYI file nicely
text = autoflake.fix_code(text,
remove_unused_variables=True,
remove_all_unused_imports=True)
# import autopep8
# text = autopep8.fix_code(text, options={
# 'aggressive': 0,
# 'experimental': 0,
# })
style = yapf.yapf_api.style.CreatePEP8Style()
text, _ = yapf.yapf_api.FormatCode(text,
filename='<stdin>',
style_config=style,
lines=None,
verify=False)
# print(text)
# Write output to file.
subdir = dirname(target)
if subdir and not os.path.isdir(subdir):
os.makedirs(subdir)
with open(target, 'w') as file:
file.write(text)
