def check_spelling(self, verbose):
"""Checks spelling."""
spelling_errors = []
with TemporaryDirectory() as tmp_dir, NamedTemporaryFile() as output:
build_cmd = [
"sphinx-build",
"-W", # turn warnings into errors
"--color", # do emit colored output
"-T", # show full traceback on exception
"-b", # builder to use
"spelling",
"-c",
DOCS_DIR,
"-d", # path for the cached environment and doctree files
self._doctree_dir,
self._src_dir, # path to documentation source files
tmp_dir,
]
print("Executing cmd: ",
" ".join([shlex.quote(c) for c in build_cmd]))
if not verbose:
print("The output is hidden until an error occurs.")
env = os.environ.copy()
env['AIRFLOW_PACKAGE_NAME'] = self.package_name
if self.for_production:
env['AIRFLOW_FOR_PRODUCTION'] = 'true'
completed_proc = run( # pylint: disable=subprocess-run-check
build_cmd,
cwd=self._src_dir,
env=env,
stdout=output if not verbose else None,
stderr=output if not verbose else None,
timeout=PROCESS_TIMEOUT,
)
if completed_proc.returncode != 0:
output.seek(0)
print(output.read().decode())
spelling_errors.append(
SpellingError(
file_path=None,
line_no=None,
spelling=None,
suggestion=None,
context_line=None,
message=
(f"Sphinx spellcheck returned non-zero exit status: {completed_proc.returncode}."
),
))
warning_text = ""
for filepath in glob(f"{tmp_dir}/**/*.spelling",
recursive=True):
with open(filepath) as spelling_file:
warning_text += spelling_file.read()
spelling_errors.extend(
parse_spelling_warnings(warning_text, self._src_dir))
return spelling_errors
def check_spelling(self):
"""Checks spelling."""
spelling_errors = []
with TemporaryDirectory() as tmp_dir, with_group(
f"Check spelling: {self.package_name}"):
build_cmd = [
"sphinx-build",
"-W", # turn warnings into errors
"-T", # show full traceback on exception
"-b", # builder to use
"spelling",
"-c",
DOCS_DIR,
"-d", # path for the cached environment and doctree files
self._doctree_dir,
self._src_dir, # path to documentation source files
tmp_dir,
]
print("Executing cmd: ",
" ".join([shlex.quote(c) for c in build_cmd]))
env = os.environ.copy()
env['AIRFLOW_PACKAGE_NAME'] = self.package_name
if self.for_production:
env['AIRFLOW_FOR_PRODUCTION'] = 'true'
completed_proc = run( # pylint: disable=subprocess-run-check
build_cmd,
cwd=self._src_dir,
env=env)
if completed_proc.returncode != 0:
spelling_errors.append(
SpellingError(
file_path=None,
line_no=None,
spelling=None,
suggestion=None,
context_line=None,
message=
(f"Sphinx spellcheck returned non-zero exit status: {completed_proc.returncode}."
),
))
warning_text = ""
for filepath in glob(f"{tmp_dir}/**/*.spelling",
recursive=True):
with open(filepath) as speeling_file:
warning_text += speeling_file.read()
spelling_errors.extend(
parse_spelling_warnings(warning_text, self._src_dir))
return spelling_errors
def check_spelling(self, verbose: bool) -> List[SpellingError]:
"""
Checks spelling
:param verbose: whether to show output while running
:return: list of errors
"""
spelling_errors = []
os.makedirs(self._build_dir, exist_ok=True)
shutil.rmtree(self.log_spelling_output_dir, ignore_errors=True)
os.makedirs(self.log_spelling_output_dir, exist_ok=True)
build_cmd = [
os.path.join(ROOT_PROJECT_DIR, "docs", "exts", "docs_build",
"run_patched_sphinx.py"),
"-W", # turn warnings into errors
"--color", # do emit colored output
"-T", # show full traceback on exception
"-b", # builder to use
"spelling",
"-c",
DOCS_DIR,
"-d", # path for the cached environment and doctree files
self._doctree_dir,
self._src_dir, # path to documentation source files
self.log_spelling_output_dir,
]
env = os.environ.copy()
env['AIRFLOW_PACKAGE_NAME'] = self.package_name
if self.for_production:
env['AIRFLOW_FOR_PRODUCTION'] = 'true'
if verbose:
console.print(
f"[blue]{self.package_name:60}:[/] Executing cmd: ",
" ".join([shlex.quote(c) for c in build_cmd]),
)
console.print(
f"[blue]{self.package_name:60}:[/] The output is hidden until an error occurs."
)
with open(self.log_spelling_filename, "wt") as output:
completed_proc = run( # pylint: disable=subprocess-run-check
build_cmd,
cwd=self._src_dir,
env=env,
stdout=output if not verbose else None,
stderr=output if not verbose else None,
timeout=PROCESS_TIMEOUT,
)
if completed_proc.returncode != 0:
spelling_errors.append(
SpellingError(
file_path=None,
line_no=None,
spelling=None,
suggestion=None,
context_line=None,
message=
(f"Sphinx spellcheck returned non-zero exit status: {completed_proc.returncode}."
),
))
warning_text = ""
for filepath in glob(
f"{self.log_spelling_output_dir}/**/*.spelling",
recursive=True):
with open(filepath) as spelling_file:
warning_text += spelling_file.read()
spelling_errors.extend(
parse_spelling_warnings(warning_text, self._src_dir))
console.print(
f"[blue]{self.package_name:60}:[/] [red]Finished spell-checking with errors[/]"
)
else:
if spelling_errors:
console.print(
f"[blue]{self.package_name:60}:[/] [yellow]Finished spell-checking with warnings[/]"
)
else:
console.print(
f"[blue]{self.package_name:60}:[/] [green]Finished spell-checking successfully[/]"
)
return spelling_errors
def check_spelling(self, verbose: bool, dockerized: bool) -> List[SpellingError]:
"""
Checks spelling
:param verbose: whether to show output while running
:param dockerized: whether to run dockerized build (required for paralllel processing on CI)
:return: list of errors
"""
spelling_errors = []
os.makedirs(self._build_dir, exist_ok=True)
shutil.rmtree(self.log_spelling_output_dir, ignore_errors=True)
os.makedirs(self.log_spelling_output_dir, exist_ok=True)
if dockerized:
python_version = os.getenv('PYTHON_MAJOR_MINOR_VERSION', "3.6")
build_cmd = [
"docker",
"run",
"--rm",
"-e",
"AIRFLOW_FOR_PRODUCTION",
"-e",
"AIRFLOW_PACKAGE_NAME",
"-v",
f"{self._build_dir}:{self._docker_build_dir}",
"-v",
f"{self._inventory_cache_dir}:{self._docker_inventory_cache_dir}",
"-w",
DOCKER_DOCS_DIR,
f"apache/airflow:master-python{python_version}-ci",
"/opt/airflow/scripts/in_container/run_anything.sh",
]
else:
build_cmd = []
build_cmd.extend(
[
"sphinx-build",
"-W", # turn warnings into errors
"--color", # do emit colored output
"-T", # show full traceback on exception
"-b", # builder to use
"spelling",
"-c",
DOCS_DIR if not dockerized else DOCKER_DOCS_DIR,
"-d", # path for the cached environment and doctree files
self._doctree_dir if not dockerized else self._docker_doctree_dir,
self._src_dir
if not dockerized
else self._docker_src_dir, # path to documentation source files
self.log_spelling_output_dir if not dockerized else self.docker_log_spelling_output_dir,
]
)
env = os.environ.copy()
env['AIRFLOW_PACKAGE_NAME'] = self.package_name
if self.for_production:
env['AIRFLOW_FOR_PRODUCTION'] = 'true'
if verbose:
console.print(
f"[blue]{self.package_name:60}:[/] Executing cmd: ",
" ".join([shlex.quote(c) for c in build_cmd]),
)
console.print(f"[blue]{self.package_name:60}:[/] The output is hidden until an error occurs.")
with open(self.log_spelling_filename, "wt") as output:
completed_proc = run( # pylint: disable=subprocess-run-check
build_cmd,
cwd=self._src_dir,
env=env,
stdout=output if not verbose else None,
stderr=output if not verbose else None,
timeout=PROCESS_TIMEOUT,
)
if completed_proc.returncode != 0:
spelling_errors.append(
SpellingError(
file_path=None,
line_no=None,
spelling=None,
suggestion=None,
context_line=None,
message=(
f"Sphinx spellcheck returned non-zero exit status: " f"{completed_proc.returncode}."
),
)
)
warning_text = ""
for filepath in glob(f"{self.log_spelling_output_dir}/**/*.spelling", recursive=True):
with open(filepath) as spelling_file:
warning_text += spelling_file.read()
spelling_errors.extend(parse_spelling_warnings(warning_text, self._src_dir, dockerized))
console.print(f"[blue]{self.package_name:60}:[/] [red]Finished spell-checking with errors[/]")
else:
if spelling_errors:
console.print(
f"[blue]{self.package_name:60}:[/] [yellow]Finished spell-checking " f"with warnings[/]"
)
else:
console.print(
f"[blue]{self.package_name:60}:[/] [green]Finished spell-checking " f"successfully[/]"
)
return spelling_errors
