def run_autopep(filename):
cmd_args = ['dummy', '-d']
args = autopep8.parse_args(cmd_args)
args.aggressive = 2
args.in_place = True
args.diff = False
args.max_line_length = 66
autopep8.fix_file(filename, args)
if not any(x in filename for x in import_fixed) or "internal" in filename:
isort.SortImports(filename)
run_autoflake(filename)
def nblint_one(nb_node):
"""format/lint one notebook"""
changes = 0
has_empty = 0
nb_metadata_keys = list(nb_node.metadata.keys())
for key in nb_metadata_keys:
if key not in NB_METADATA_KEYS:
nb_node.metadata.pop(key)
for cell in nb_node.cells:
cell_type = cell["cell_type"]
source = "".join(cell["source"])
if not source.strip():
has_empty += 1
if cell_type == "markdown":
args = [
*P.PRETTIER,
"--stdin-filepath",
"foo.md",
"--prose-wrap",
"always",
]
prettier = subprocess.Popen(
list(map(str, args)),
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
)
out, _err = prettier.communicate(source.encode("utf-8"))
new = out.decode("utf-8").rstrip()
if new != source:
cell["source"] = new.splitlines(True)
changes += 1
elif cell_type == "code":
if cell["outputs"] or cell["execution_count"]:
cell["outputs"] = []
cell["execution_count"] = None
changes += 1
if [line for line in source.splitlines() if line.strip().startswith("!")]:
continue
if source.startswith("%"):
continue
new = isort.SortImports(file_contents=source).output
new = blacken(new).rstrip()
if new != source:
cell["source"] = new.splitlines(True)
changes += 1
if has_empty:
changes += 1
nb_node.cells = [
cell for cell in nb_node.cells if "".join(cell["source"]).strip()
]
return nb_node
def test_import_order():
"""All source file imports should be ordered and formatted properly."""
file_paths = itertools.chain(
glob.iglob('automata/*/*.py'),
glob.iglob('tests/*.py'))
for file_path in file_paths:
with open(file_path, 'r') as file_obj:
file_contents = file_obj.read()
len_change = isort.SortImports(
file_contents=file_contents).length_change
fail_msg = '{} imports do not comply with PEP 8'.format(file_path)
yield nose.assert_equal, len_change, 0, fail_msg
def __init__(self, linter):
"""Initialize the linter by loading all 'allowed' imports from package requirements"""
super().__init__(linter)
self.known_files = {} # type: Dict[str, _DistInfo]
self.known_modules = defaultdict(set) # type: defaultdict[str, Set[_DistInfo]]
if hasattr(isort, "place_module"): # isort >= v5
self._isort_place_module = isort.place_module # pylint: disable=no-member
else:
sorter = isort.SortImports(file_contents="") # pylint: disable=no-member
self._isort_place_module = sorter.place_module
all_loadable_distributions = set(
importlib_metadata.distributions()
) # type: Set[Distribution]
setup_result = run_setup("setup.py")
self.first_party_packages = _filter_non_namespace_packages(setup_result.packages or [])
self.allowed_distributions = {
get_distribution(x).project_name for x in setup_result.install_requires
}
self.visited_distributions = set()
self.dists_without_file_info = set()
for dist in all_loadable_distributions:
dist_name = dist.metadata["Name"]
allowed = dist_name in self.allowed_distributions
# Get a list of files created by the distribution
distribution_files = dist.files or []
# Resolve the (relative) paths to absolute paths
resolved_filepaths = {x.locate() for x in distribution_files}
# in python3.4 dict.get() always returns None when passing a pathlib.Path as key
distribution_file_info = {
str(p): _DistInfo(dist, allowed) for p in resolved_filepaths
}
# Add them to the whitelist
self.known_files.update(distribution_file_info)
# Add distributions without files to candidate list for unmatched imports
if not distribution_file_info:
self.dists_without_file_info.add(dist_name)
# Add source distributions to backup list
if not dist.read_text("SOURCES.txt"):
continue
dist_modules_text = dist.read_text("top_level.txt") or ""
dist_modules = dist_modules_text.split()
for mod in dist_modules:
self.known_modules[mod].add(_DistInfo(dist, allowed))
def run(self):
try:
import isort
except ImportError:
print(('Cannot import isort, you forgot to install?\n'
'run `pip install isort` to install.'),
file=sys.stderr)
sys.exit(1)
from glob import glob
print()
print('Options')
print('=======')
print()
print('Exclude:', EXCLUDE_SCRIPTS)
print()
files = ['setup.py', CLI_script, module_file] + glob('tests/*.py')
print('Results')
print('=======')
print()
fails = 0
for f in files:
# unfortunately, we have to do it twice
if isort.SortImports(f, check=True).incorrectly_sorted:
fails += 1
print()
isort.SortImports(f, show_diff=True)
print()
print()
print('Statistics')
print('==========')
print()
print('%d files failed to pass' % fails)
def _check_imports_order(self, _module_node):
"""Checks imports of module `node` are grouped by category
Imports must follow this order: standard, 3rd party, local
"""
extern_imports = []
local_imports = []
std_imports = []
extern_not_ignored = []
local_not_ignored = []
isort_obj = isort.SortImports(
file_contents='',
known_third_party=self.config.known_third_party,
known_standard_library=self.config.known_standard_library,
)
for node, modname in self._imports_stack:
if modname.startswith('.'):
package = '.' + modname.split('.')[1]
else:
package = modname.split('.')[0]
nested = not isinstance(node.parent, astroid.Module)
ignore_for_import_order = not self.linter.is_message_enabled(
'wrong-import-order', node.fromlineno)
import_category = isort_obj.place_module(package)
if import_category in ('FUTURE', 'STDLIB'):
std_imports.append((node, package))
wrong_import = extern_not_ignored or local_not_ignored
if self._is_fallback_import(node, wrong_import):
continue
if wrong_import and not nested:
self.add_message(
'wrong-import-order',
node=node,
args=('standard import "%s"' % node.as_string(),
'"%s"' % wrong_import[0][0].as_string()))
elif import_category in ('FIRSTPARTY', 'THIRDPARTY'):
extern_imports.append((node, package))
if not nested and not ignore_for_import_order:
extern_not_ignored.append((node, package))
wrong_import = local_not_ignored
if wrong_import and not nested:
self.add_message(
'wrong-import-order',
node=node,
args=('external import "%s"' % node.as_string(),
'"%s"' % wrong_import[0][0].as_string()))
elif import_category == 'LOCALFOLDER':
local_imports.append((node, package))
if not nested and not ignore_for_import_order:
local_not_ignored.append((node, package))
return std_imports, extern_imports, local_imports
def __init__(self, config):
if HAS_ISORT_5:
self.isort5_config = isort.api.Config(
# There is not typo here. EXTRA_standard_library is
# what most users want. The option has been named
# KNOWN_standard_library for ages in pylint and we
# don't want to break compatibility.
extra_standard_library=config.known_standard_library,
known_third_party=config.known_third_party,
)
else:
self.isort4_obj = isort.SortImports( # pylint: disable=no-member
file_contents="",
known_standard_library=config.known_standard_library,
known_third_party=config.known_third_party,
)
def _process_request(self, request):
'''Processes json request, writes appropriate response.
eg: If the `request['type']` is 'sort_text', write a json dict
containing a sorted version of `request['file_contents']` via
`self._write_response`.
Args:
request (str): A string representing a json dump made by this
script must have 'file_contents', 'file_path', and 'type' keys.
'''
request = json.loads(request)
with silence_stdout():
new_contents = isort.SortImports(
file_contents=request['file_contents'],
file_path=request.get('file_path'),
write_to_stdout=True,
not_skip=['__init__.py', request.get('file_path')])
if hasattr(new_contents, 'output'):
new_contents = new_contents.output
else:
return self._write_response(
self._serialize_response('error'), {'description': 'no output from isort'})
if request['type'] == 'sort_text':
return self._write_response(
self._serialize_response('sort_text_response', {'new_contents': new_contents}))
if request['type'] == 'check_text':
# NOTE: Some explanation required:
# Since we are using stdout, we can't use the default 'check=True'
# option, since isort will write to sdtout if there are errors,
# and this cannot be overridden with a function call. However, we
# can replicate the behavior by sorting imports and then comparing
# to the unsorted text. If they are different, then they are not
# sorted.
if len(request['file_contents'].split()) == 0:
correctly_sorted = True
else:
correctly_sorted = (new_contents == request['file_contents'])
return self._write_response(
self._serialize_response('check_text_response', {
'correctly_sorted': correctly_sorted,
}))
def format_file(file: str) -> None:
file_path = Path(file)
if file_path.suffix in PYTHON_FILE_EXTENSIONS:
mode = black.FileMode(
target_versions={black.TargetVersion.PY38},
line_length=LINE_LENGTH,
is_pyi=file_path.suffix == ".pyi",
string_normalization=True,
)
isort.SortImports(file_path)
black.format_file_in_place(file_path, False, mode, black.WriteBack.YES)
elif file_path.suffix in CXX_FILE_EXTENSIONS:
subprocess.run(["clang-format", "-i", "-style=file", file_path],
cwd=WORKSPACE)
elif file_path.suffix == ".bzl" or file_path.name in {
"BUILD", "WORKSPACE"
}:
subprocess.run([BUILDIFIER, "-lint", "fix", file_path], cwd=WORKSPACE)
def run(self):
if self.filename is not self.stdin_display_name:
file_path = self.filename
else:
file_path = None
with OutputCapture() as buffer:
sort_result = isort.SortImports(
file_path=file_path,
file_contents=''.join(self.lines),
check=True,
show_diff=True,
)
traceback = self._format_isort_output(buffer)
for line_num, message in self.sortimports_linenum_msg(sort_result):
if self.show_traceback:
message += traceback
yield line_num, 0, message, type(self)
def _check_imported_packages(self, node, module_name):
"""Check if the import node is a external dependency to validate it"""
if not module_name:
# skip local packages because is not a external dependency.
return
if not self.manifest_dict:
# skip if is not a module of odoo
return
if not isinstance(node.parent, astroid.Module):
# skip nested import sentences
return
if self._is_absolute_import(node, module_name):
# skip absolute imports
return
if self._is_module_name_in_whitelist(module_name):
# ignore whitelisted modules
return
isort_obj = isort.SortImports(file_contents='')
import_category = isort_obj.place_module(module_name)
if import_category not in ('FIRSTPARTY', 'THIRDPARTY'):
# skip if is not a external library or is a white list library
return
relpath = os.path.relpath(node.parent.file,
os.path.dirname(self.manifest_file))
if os.path.dirname(relpath) == 'tests':
# import errors rules don't apply to the test files
# since these files are loaded only when running tests
# and in such a case your
# module and their external dependencies are installed.
return
self.add_message('missing-import-error',
node=node,
args=(module_name, ))
ext_deps = self.manifest_dict.get('external_dependencies') or {}
py_ext_deps = ext_deps.get('python') or []
if isinstance(node, astroid.ImportFrom) and (node.level or 0) >= 1:
return
if module_name not in py_ext_deps and \
module_name.split('.')[0] not in py_ext_deps and \
not any(dep in module_name for dep in py_ext_deps):
self.add_message('missing-manifest-dependency',
node=node,
args=(module_name, ))
def _check_import_order(self):
"""This function is used to check that each file
has imports placed in alphabetical order.
"""
if self.verbose_mode_enabled:
python_utils.PRINT('Starting import-order checks')
python_utils.PRINT('----------------------------------------')
summary_messages = []
files_to_check = self.all_filepaths
failed = False
stdout = python_utils.string_io()
with linter_utils.redirect_stdout(stdout):
for filepath in files_to_check:
# This line prints the error message along with file path
# and returns True if it finds an error else returns False
# If check is set to True, isort simply checks the file and
# if check is set to False, it autocorrects import-order errors.
if (isort.SortImports(filepath, check=True,
show_diff=(True)).incorrectly_sorted):
failed = True
python_utils.PRINT('')
python_utils.PRINT('')
if failed:
summary_message = stdout.getvalue()
summary_messages.append(summary_message)
summary_message = (
'%s Import order checks failed, file imports should be '
'alphabetized, see affect files above.' %
(linter_utils.FAILED_MESSAGE_PREFIX))
python_utils.PRINT(summary_message)
summary_messages.append(summary_message)
else:
summary_message = ('%s Import order checks passed' %
(linter_utils.SUCCESS_MESSAGE_PREFIX))
python_utils.PRINT(summary_message)
summary_messages.append(summary_message)
return summary_messages
def render_template(
self, template_name: str, context: Dict[str, Any], reformat_code: bool = True
):
log.info("Render template %r with context %s", template_name, context)
code = self.env.get_template(template_name).render(context)
if reformat_code:
code = autoflake.fix_code(
code,
additional_imports=None,
expand_star_imports=True,
remove_all_unused_imports=True,
remove_duplicate_keys=True,
remove_unused_variables=False,
ignore_init_module_imports=False,
)
code = isort.SortImports(file_contents=code).output
try:
code = black.format_file_contents(
code,
fast=True,
mode=black.FileMode(
target_versions={black.TargetVersion.PY37}, line_length=99
),
)
except black.NothingChanged:
pass
except black.InvalidInput:
print(code)
raise
def fix_line(line: str) -> str:
if not line.strip():
line = ''
return line
code = '\n'.join(map(fix_line, code.split('\n'))).strip() + '\n'
return code
def isort_notebook_cells(notebook):
with open(notebook, "rb") as fp:
nb = nbformat.read(fp=fp, as_version=nbformat.NO_CONVERT)
markdown_cells = list()
code_cells = list()
for cell in nb["cells"]:
if cell["cell_type"] == "code":
code_cells.append(cell)
elif cell["cell_type"] == "markdown":
markdown_cells.append(cell)
else:
raise Exception(cell["cell_type"])
for code_cell in code_cells:
if code_cell["source"] == "":
continue
if "import" in code_cell["source"]:
s = isort.SortImports(file_contents=code_cell.source)
code_cell.source = s.output.strip()
with open(notebook, "w") as fp:
nbformat.write(nb, fp)
def _check_imports_order(self, _module_node):
"""Checks imports of module `node` are grouped by category
Imports must follow this order: standard, 3rd party, local
"""
std_imports = []
third_party_imports = []
first_party_imports = []
# need of a list that holds third or first party ordered import
external_imports = []
local_imports = []
third_party_not_ignored = []
first_party_not_ignored = []
local_not_ignored = []
isort_obj = isort.SortImports(
file_contents="",
known_third_party=self.config.known_third_party,
known_standard_library=self.config.known_standard_library,
)
for node, modname in self._imports_stack:
if modname.startswith("."):
package = "." + modname.split(".")[1]
else:
package = modname.split(".")[0]
nested = not isinstance(node.parent, astroid.Module)
ignore_for_import_order = not self.linter.is_message_enabled(
"wrong-import-order", node.fromlineno)
import_category = isort_obj.place_module(package)
node_and_package_import = (node, package)
if import_category in ("FUTURE", "STDLIB"):
std_imports.append(node_and_package_import)
wrong_import = (third_party_not_ignored
or first_party_not_ignored
or local_not_ignored)
if self._is_fallback_import(node, wrong_import):
continue
if wrong_import and not nested:
self.add_message(
"wrong-import-order",
node=node,
args=(
'standard import "%s"' % node.as_string(),
'"%s"' % wrong_import[0][0].as_string(),
),
)
elif import_category == "THIRDPARTY":
third_party_imports.append(node_and_package_import)
external_imports.append(node_and_package_import)
if not nested and not ignore_for_import_order:
third_party_not_ignored.append(node_and_package_import)
wrong_import = first_party_not_ignored or local_not_ignored
if wrong_import and not nested:
self.add_message(
"wrong-import-order",
node=node,
args=(
'third party import "%s"' % node.as_string(),
'"%s"' % wrong_import[0][0].as_string(),
),
)
elif import_category == "FIRSTPARTY":
first_party_imports.append(node_and_package_import)
external_imports.append(node_and_package_import)
if not nested and not ignore_for_import_order:
first_party_not_ignored.append(node_and_package_import)
wrong_import = local_not_ignored
if wrong_import and not nested:
self.add_message(
"wrong-import-order",
node=node,
args=(
'first party import "%s"' % node.as_string(),
'"%s"' % wrong_import[0][0].as_string(),
),
)
elif import_category == "LOCALFOLDER":
local_imports.append((node, package))
if not nested and not ignore_for_import_order:
local_not_ignored.append((node, package))
return std_imports, external_imports, local_imports
def add_import(self, source, new_import):
source = isort.SortImports(file_contents=source,
add_imports=(new_import, ),
settings_path=self.settings_path).output
self.write(file=source)
def apply_isort(code: str) -> str:
if isort.__version__.startswith('4.'):
return isort.SortImports(file_contents=code).output
else:
return isort.code(code)
def __init__(self, linter=None):
BaseChecker.__init__(self, linter)
self.isort_obj = isort.SortImports(file_contents='', )
def apply_isort(self, code: str) -> str:
return isort.SortImports(file_contents=code,
settings_path=self.settings_path).output
def __init__(self):
if HAS_ISORT_5:
self.isort5_config = isort.api.Config()
else:
self.isort4_obj = isort.SortImports( # pylint: disable=no-member
file_contents="")
def format_string(cls, old_contents):
"""Format content of a file."""
new_contents = isort.SortImports(file_contents=old_contents).output
return old_contents, new_contents, 'utf-8'
def __init__(self, linter: Optional[PyLinter] = None) -> None:
BaseChecker.__init__(self, linter)
self.isort_obj = isort.SortImports(file_contents='', )
def isort_cell(cell):
cell.source = isort.SortImports(
file_contents=cell.source, setting_overrides=_ISORT_SETTINGS
).output.strip()