def main(args=None):
"""Runs the MyPy type checks."""
parsed_args = _PARSER.parse_args(args=args)
for directory in common.DIRS_TO_ADD_TO_SYS_PATH:
# The directories should only be inserted starting at index 1. See
# https://stackoverflow.com/a/10095099 and
# https://stackoverflow.com/q/10095037 for more details.
sys.path.insert(1, directory)
install_third_party_libraries(parsed_args.skip_install)
common.fix_third_party_imports()
python_utils.PRINT('Installing Mypy and stubs for third party libraries.')
return_code = install_mypy_prerequisites()
if return_code != 0:
python_utils.PRINT(
'Cannot install Mypy and stubs for third party libraries.')
sys.exit(1)
python_utils.PRINT('Installed Mypy and stubs for third party libraries.')
python_utils.PRINT('Starting Mypy type checks.')
cmd = get_mypy_cmd(getattr(parsed_args, 'files'))
process = subprocess.call(cmd, stdin=subprocess.PIPE)
if process == 0:
python_utils.PRINT('Mypy type checks successful.')
else:
python_utils.PRINT(
'Mypy type checks unsuccessful. Please fix the errors.')
sys.exit(1)
return process
def main(args=None):
"""Runs the MyPy type checks."""
parsed_args = _PARSER.parse_args(args=args)
for directory in common.DIRS_TO_ADD_TO_SYS_PATH:
# The directories should only be inserted starting at index 1. See
# https://stackoverflow.com/a/10095099 and
# https://stackoverflow.com/q/10095037 for more details.
sys.path.insert(1, directory)
install_third_party_libraries(parsed_args.skip_install)
common.fix_third_party_imports()
python_utils.PRINT('Installing Mypy and stubs for third party libraries.')
return_code, mypy_exec_path = install_mypy_prerequisites(
parsed_args.install_globally)
if return_code != 0:
python_utils.PRINT(
'Cannot install Mypy and stubs for third party libraries.')
sys.exit(1)
python_utils.PRINT('Installed Mypy and stubs for third party libraries.')
python_utils.PRINT('Starting Mypy type checks.')
cmd = get_mypy_cmd(parsed_args.files, mypy_exec_path,
parsed_args.install_globally)
env = os.environ.copy()
for path in _PATHS_TO_INSERT:
env['PATH'] = '%s%s' % (path, os.pathsep) + env['PATH']
env['PYTHONPATH'] = MYPY_TOOLS_DIR
process = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=env)
stdout, stderr = process.communicate()
# Standard and error output is in bytes, we need to decode the line to
# print it.
python_utils.PRINT(stdout.decode('utf-8'))
python_utils.PRINT(stderr.decode('utf-8'))
if process.returncode == 0:
python_utils.PRINT('Mypy type checks successful.')
else:
python_utils.PRINT(
'Mypy type checks unsuccessful. Please fix the errors. '
'For more information, visit: '
'https://github.com/oppia/oppia/wiki/Backend-Type-Annotations')
sys.exit(2)
return process.returncode
def main(args=None):
"""Main method for pre commit linter script that lints Python, JavaScript,
HTML, and CSS files.
"""
# Namespace is used to share values between multiple processes. This cannot
# be used as a global variable since then it leads to hanging of some
# processes.
namespace = multiprocessing.Manager().Namespace()
parsed_args = _PARSER.parse_args(args=args)
# File extension to be linted.
file_extension_types = _get_file_extensions(
parsed_args.only_check_file_extensions)
# Default mode is non-verbose mode, if arguments contains --verbose flag it
# will be made True, which will represent verbose mode.
verbose_mode_enabled = bool(parsed_args.verbose)
all_filepaths = _get_all_filepaths(
parsed_args.path,
parsed_args.files,
parsed_args.shard,
namespace=namespace
)
install_third_party_libs.main()
common.fix_third_party_imports()
print('Starting Linter....')
if len(all_filepaths) == 0:
print('---------------------------')
print('No files to check.')
print('---------------------------')
return
read_files(all_filepaths, namespace=namespace)
files = multiprocessing.Manager().dict()
categorize_files(all_filepaths, files)
# Prepare custom tasks.
custom_max_concurrent_runs = 25
custom_concurrent_count = min(
multiprocessing.cpu_count(), custom_max_concurrent_runs)
custom_semaphore = threading.Semaphore(custom_concurrent_count)
# Prepare third_party tasks.
third_party_max_concurrent_runs = 2
third_party_concurrent_count = min(
multiprocessing.cpu_count(), third_party_max_concurrent_runs)
third_party_semaphore = threading.Semaphore(third_party_concurrent_count)
custom_linters = []
third_party_linters = []
for file_extension_type in file_extension_types:
if file_extension_type in ('js', 'ts'):
if len(files['.js'] + files['.ts']) == 0:
continue
elif (not file_extension_type == 'other' and not
len(files['.%s' % file_extension_type])):
continue
custom_linter, third_party_linter = _get_linters_for_file_extension(
file_extension_type, namespace, files)
custom_linters += custom_linter
third_party_linters += third_party_linter
# Create tasks.
tasks_custom = []
tasks_third_party = []
for linter in custom_linters:
name = _get_space_separated_linter_name(type(linter).__name__)
task_custom = concurrent_task_utils.create_task(
linter.perform_all_lint_checks, verbose_mode_enabled,
custom_semaphore, name=name)
tasks_custom.append(task_custom)
for linter in third_party_linters:
name = _get_space_separated_linter_name(type(linter).__name__)
task_third_party = concurrent_task_utils.create_task(
linter.perform_all_lint_checks, verbose_mode_enabled,
third_party_semaphore, name=name)
tasks_third_party.append(task_third_party)
# Execute tasks.
# Here we set Concurrency limit for custom task to 25 because we need to
# parallelize the tasks to work on full capacity of CPU.
# Concurrency limit for third party tasks is set to 2 because these
# third party libraries have their own ways to lint at their fastest
# (ie. might parallelize on their own)
# Concurrency limit: 25.
concurrent_task_utils.execute_tasks(tasks_custom, custom_semaphore)
# Concurrency limit: 2.
concurrent_task_utils.execute_tasks(
tasks_third_party, third_party_semaphore)
lint_messages = []
failed = False
for task in tasks_custom:
failed = _get_task_output(lint_messages, failed, task)
for task in tasks_third_party:
failed = _get_task_output(lint_messages, failed, task)
errors_stacktrace = concurrent_task_utils.ALL_ERRORS
if errors_stacktrace:
failed = True
_print_errors_stacktrace(errors_stacktrace)
if failed:
_print_summary_of_error_messages(lint_messages)
linter_utils.print_failure_message('\n'.join([
'---------------------------',
'Checks Not Passed.',
'---------------------------']))
sys.exit(1)
else:
linter_utils.print_success_message('\n'.join([
'---------------------------',
'All Checks Passed.',
'---------------------------']))
