def test_file_invalid(self):
with retrieve_stdout() as sio:
self.assertEqual(collect_files(["invalid_path"],
self.log_printer), [])
self.assertRegex(sio.getvalue(),
".*\\[WARNING\\].*No files matching "
"'invalid_path' were found.\n")
def _instantiate_processes(self, job_count):
filename_list = collect_files(path_list(self.section.get('files', "")))
file_dict = self._get_file_dict(filename_list)
manager = multiprocessing.Manager()
global_bear_queue = multiprocessing.Queue()
filename_queue = multiprocessing.Queue()
local_result_dict = manager.dict()
global_result_dict = manager.dict()
message_queue = multiprocessing.Queue()
control_queue = multiprocessing.Queue()
barrier = Barrier(parties=job_count)
bear_runner_args = {"file_name_queue": filename_queue,
"local_bear_list": self.local_bear_list,
"global_bear_list": self.global_bear_list,
"global_bear_queue": global_bear_queue,
"file_dict": file_dict,
"local_result_dict": local_result_dict,
"global_result_dict": global_result_dict,
"message_queue": message_queue,
"control_queue": control_queue,
"barrier": barrier,
"TIMEOUT": 0.1}
self._instantiate_bears(file_dict,
message_queue)
self._fill_queue(filename_queue, filename_list)
self._fill_queue(global_bear_queue, range(len(self.global_bear_list)))
return ([BearRunner(**bear_runner_args) for i in range(job_count)],
bear_runner_args)
def test_file_collection(self):
self.assertEqual(collect_files([os.path.join(self.collectors_test_dir,
"others",
"*",
"*2.py")]),
[os.path.join(self.collectors_test_dir,
"others",
"py_files",
"file2.py")])
def instantiate_processes(section,
local_bear_list,
global_bear_list,
job_count,
log_printer):
"""
Instantiate the number of processes that will run bears which will be
responsible for running bears in a multiprocessing environment.
:param section: The section the bears belong to.
:param local_bear_list: List of local bears belonging to the section.
:param global_bear_list: List of global bears belonging to the section.
:param job_count: Max number of processes to create.
:param log_printer: The log printer to warn to.
:return: A tuple containing a list of processes,
and the arguments passed to each process which are
the same for each object.
"""
filename_list = collect_files(
path_list(section.get('files', "")),
log_printer,
ignored_file_paths=path_list(section.get('ignore', "")),
limit_file_paths=path_list(section.get('limit_files', "")))
file_dict = get_file_dict(filename_list, log_printer)
manager = multiprocessing.Manager()
global_bear_queue = multiprocessing.Queue()
filename_queue = multiprocessing.Queue()
local_result_dict = manager.dict()
global_result_dict = manager.dict()
message_queue = multiprocessing.Queue()
control_queue = multiprocessing.Queue()
bear_runner_args = {"file_name_queue": filename_queue,
"local_bear_list": local_bear_list,
"global_bear_list": global_bear_list,
"global_bear_queue": global_bear_queue,
"file_dict": file_dict,
"local_result_dict": local_result_dict,
"global_result_dict": global_result_dict,
"message_queue": message_queue,
"control_queue": control_queue,
"timeout": 0.1}
local_bear_list[:], global_bear_list[:] = instantiate_bears(
section,
local_bear_list,
global_bear_list,
file_dict,
message_queue)
fill_queue(filename_queue, file_dict.keys())
fill_queue(global_bear_queue, range(len(global_bear_list)))
return ([multiprocessing.Process(target=run, kwargs=bear_runner_args)
for i in range(job_count)],
bear_runner_args)
def test_file_string_collection(self):
self.assertEqual(collect_files(os.path.join(self.collectors_test_dir,
"others",
"*",
"*2.py")),
[os.path.normcase(os.path.join(
self.collectors_test_dir,
"others",
"py_files",
"file2.py"))])
def test_trailing_globstar(self):
ignore_path1 = os.path.join(self.collectors_test_dir,
'others',
'c_files',
'**') # should generate warning
ignore_path2 = os.path.join(self.collectors_test_dir,
'**',
'py_files',
'**') # no warning
with LogCapture() as capture:
collect_files(file_paths=[],
ignored_file_paths=[ignore_path1, ignore_path2],
log_printer=self.log_printer)
capture.check(
('root', 'WARNING', 'Detected trailing globstar in ignore glob '
'\'{}\'. Please remove the unnecessary \'**\''
' from its end.'
.format(ignore_path1))
)
def test_file_invalid(self):
with LogCapture() as capture:
self.assertEqual(collect_files(file_paths=['invalid_path'],
log_printer=self.log_printer,
section_name='section'), [])
capture.check(
('root', 'WARNING', 'No files matching \'invalid_path\' were '
'found. If this rule is not required, you can '
'remove it from section [section] in your '
'.coafile to deactivate this warning.')
)
def test_file_string_collection(self):
self.assertEqual(collect_files(os.path.join(self.collectors_test_dir,
'others',
'*',
'*2.py'),
self.log_printer),
[os.path.normcase(os.path.join(
self.collectors_test_dir,
'others',
'py_files',
'file2.py'))])
def test_file_invalid(self):
self.assertEqual(collect_files(file_paths=['invalid_path'],
log_printer=self.log_printer,
ignored_file_paths=None,
limit_file_paths=None,
section_name='section'), [])
self.assertEqual([log.message for log in self.log_printer.logs],
['No files matching \'invalid_path\' were found. '
'If this rule is not required, you can remove it '
'from section [section] in your .coafile to '
'deactivate this warning.'
])
def test_file_invalid(self):
with LogCapture() as capture:
self.assertEqual(
collect_files(file_paths=['invalid_path'],
log_printer=self.log_printer,
ignored_file_paths=None,
limit_file_paths=None,
section_name='section'), [])
capture.check(
('root', 'WARNING', 'No files matching \'invalid_path\' were '
'found. If this rule is not required, you can '
'remove it from section [section] in your '
'.coafile to deactivate this warning.'))
def test_limited(self):
self.assertEqual(
collect_files(
[os.path.join(self.collectors_test_dir, "others", "*", "*py")],
self.log_printer,
limit_file_paths=[
os.path.join(self.collectors_test_dir, "others", "*",
"*2.py")
]), [
os.path.normcase(
os.path.join(self.collectors_test_dir, "others",
"py_files", "file2.py"))
])
def instantiate_processes(section, local_bear_list, global_bear_list,
job_count, log_printer):
"""
Instantiate the number of processes that will run bears which will be
responsible for running bears in a multiprocessing environment.
:param section: The section the bears belong to.
:param local_bear_list: List of local bears belonging to the section.
:param global_bear_list: List of global bears belonging to the section.
:param job_count: Max number of processes to create.
:param log_printer: The log printer to warn to.
:return: A tuple containing a list of processes,
and the arguments passed to each process which are
the same for each object.
"""
filename_list = collect_files(path_list(section.get('files', "")),
path_list(section.get('ignore', "")))
file_dict = get_file_dict(filename_list, log_printer)
manager = multiprocessing.Manager()
global_bear_queue = multiprocessing.Queue()
filename_queue = multiprocessing.Queue()
local_result_dict = manager.dict()
global_result_dict = manager.dict()
message_queue = multiprocessing.Queue()
control_queue = multiprocessing.Queue()
bear_runner_args = {
"file_name_queue": filename_queue,
"local_bear_list": local_bear_list,
"global_bear_list": global_bear_list,
"global_bear_queue": global_bear_queue,
"file_dict": file_dict,
"local_result_dict": local_result_dict,
"global_result_dict": global_result_dict,
"message_queue": message_queue,
"control_queue": control_queue,
"timeout": 0.1
}
local_bear_list[:], global_bear_list[:] = instantiate_bears(
section, local_bear_list, global_bear_list, file_dict, message_queue)
fill_queue(filename_queue, file_dict.keys())
fill_queue(global_bear_queue, range(len(global_bear_list)))
return ([
multiprocessing.Process(target=run, kwargs=bear_runner_args)
for i in range(job_count)
], bear_runner_args)
def test_limited(self):
self.assertEqual(
collect_files([os.path.join(self.collectors_test_dir,
"others",
"*",
"*py")],
limit_file_paths=[os.path.join(
self.collectors_test_dir,
"others",
"*",
"*2.py")]),
[os.path.normcase(os.path.join(self.collectors_test_dir,
"others",
"py_files",
"file2.py"))])
def test_limited(self):
self.assertEqual(
collect_files([os.path.join(self.collectors_test_dir,
'others',
'*',
'*py')],
self.log_printer,
limit_file_paths=[os.path.join(
self.collectors_test_dir,
'others',
'*',
'*2.py')]),
[os.path.normcase(os.path.join(self.collectors_test_dir,
'others',
'py_files',
'file2.py'))])
def test_ignored(self):
self.assertEqual(collect_files([os.path.join(self.collectors_test_dir,
'others',
'*',
'*2.py'),
os.path.join(self.collectors_test_dir,
'others',
'*',
'*2.py')],
self.log_printer,
ignored_file_paths=[os.path.join(
self.collectors_test_dir,
'others',
'py_files',
'file2.py')]),
[])
def test_ignored(self):
self.assertEqual(collect_files([os.path.join(self.collectors_test_dir,
"others",
"*",
"*2.py"),
os.path.join(self.collectors_test_dir,
"others",
"*",
"*2.py")],
self.log_printer,
ignored_file_paths=[os.path.join(
self.collectors_test_dir,
"others",
"py_files",
"file2.py")]),
[])
def test_ignored(self):
self.assertEqual(collect_files([os.path.join(self.collectors_test_dir,
"others",
"*",
"*2.py"),
os.path.join(self.collectors_test_dir,
"others",
"*",
"*2.py")],
self.log_printer,
ignored_file_paths=[os.path.join(
self.collectors_test_dir,
"others",
"py_files",
"file2.py")]),
[])
def test_ignored(self):
self.assertEqual(collect_files([os.path.join(self.collectors_test_dir,
'others',
'*',
'*2.py'),
os.path.join(self.collectors_test_dir,
'others',
'*',
'*2.py')],
self.log_printer,
ignored_file_paths=[os.path.join(
self.collectors_test_dir,
'others',
'py_files',
'file2.py')]),
[])
def test_limited(self):
self.assertEqual(
collect_files([os.path.join(self.collectors_test_dir,
'others',
'*',
'*py')],
self.log_printer,
limit_file_paths=[os.path.join(
self.collectors_test_dir,
'others',
'*',
'*2.py')]),
[os.path.normcase(os.path.join(self.collectors_test_dir,
'others',
'py_files',
'file2.py'))])
def get_project_files(log_printer, printer, project_dir):
"""
Gets the list of files matching files in the user's project directory
after prompting for glob expressions.
:param log_printer:
A ``LogPrinter`` object.
:param printer:
A ``ConsolePrinter`` object.
:return:
A list of file paths matching the files.
"""
file_globs = ["**"]
ignore_globs = None
if os.path.isfile(os.path.join(project_dir, ".gitignore")):
printer.print(
"The contents of your .gitignore file for the project "
"will be automatically loaded as the files to ignore.",
color="green")
ignore_globs = get_gitignore_glob(project_dir)
if ignore_globs is None:
printer.print(GLOB_HELP)
ignore_globs = ask_question("Which files do you want coala to ignore?",
printer=printer,
typecast=list)
printer.print()
escaped_project_dir = glob_escape(project_dir)
file_path_globs = [
os.path.join(escaped_project_dir, glob_exp) for glob_exp in file_globs
]
ignore_path_globs = [
os.path.join(escaped_project_dir, glob_exp)
for glob_exp in ignore_globs
]
ignore_path_globs.append(os.path.join(escaped_project_dir, ".git/**"))
file_paths = collect_files(file_path_globs,
log_printer,
ignored_file_paths=ignore_path_globs)
return file_paths, ignore_globs
def get_project_files(log_printer, printer, project_dir):
"""
Gets the list of files matching files in the user's project directory
after prompting for glob expressions.
:param log_printer:
A ``LogPrinter`` object.
:param printer:
A ``ConsolePrinter`` object.
:return:
A list of file paths matching the files.
"""
printer.print(GLOB_HELP)
file_globs = ask_question(
"Which files do you want coala to run on?",
default="**",
printer=printer,
typecast=list)
ignore_globs = ask_question(
"Which files do you want coala to run on?",
printer=printer,
typecast=list)
printer.print()
escaped_project_dir = glob_escape(project_dir)
file_path_globs = [os.path.join(
escaped_project_dir, glob_exp) for glob_exp in file_globs]
ignore_path_globs = [os.path.join(
escaped_project_dir, glob_exp) for glob_exp in ignore_globs]
file_paths = collect_files(
file_path_globs,
log_printer,
ignored_file_paths=ignore_path_globs)
return file_paths
def test_file_invalid(self):
self.assertEqual(collect_files(["invalid_path"]), [])
def test_file_invalid(self):
self.assertEqual(collect_files(['invalid_path'],
self.log_printer), [])
self.assertEqual([log.message for log in self.log_printer.logs],
["No files matching 'invalid_path' were found."])
def instantiate_processes(section,
local_bear_list,
global_bear_list,
job_count,
cache,
log_printer,
console_printer,
debug=False,
use_raw_files=False,
debug_bears=False):
"""
Instantiate the number of processes that will run bears which will be
responsible for running bears in a multiprocessing environment.
:param section: The section the bears belong to.
:param local_bear_list: List of local bears belonging to the section.
:param global_bear_list: List of global bears belonging to the section.
:param job_count: Max number of processes to create.
:param cache: An instance of ``misc.Caching.FileCache`` to use as
a file cache buffer.
:param log_printer: The log printer to warn to.
:param console_printer: Object to print messages on the console.
:param debug: Bypass multiprocessing and activate debug mode
for bears, not catching any exceptions on running
them.
:param use_raw_files: Allow the usage of raw files (non text files)
:return: A tuple containing a list of processes,
and the arguments passed to each process which are
the same for each object.
"""
filename_list = collect_files(
glob_list(section.get('files', '')),
None,
ignored_file_paths=glob_list(section.get('ignore', '')),
limit_file_paths=glob_list(section.get('limit_files', '')),
section_name=section.name)
# This stores all matched files irrespective of whether coala is run
# only on changed files or not. Global bears require all the files
complete_filename_list = filename_list
file_dict_generator = get_file_dict
if cache is not None and isinstance(cache, FileDictGenerator):
file_dict_generator = cache.get_file_dict
complete_file_dict = file_dict_generator(complete_filename_list,
allow_raw_files=use_raw_files)
logging.debug('Files that will be checked:\n' +
'\n'.join(complete_file_dict.keys()))
if debug or debug_bears:
from . import DebugProcessing as processing
else:
import multiprocessing as processing
manager = processing.Manager()
global_bear_queue = processing.Queue()
filename_queue = processing.Queue()
local_result_dict = manager.dict()
global_result_dict = manager.dict()
message_queue = processing.Queue()
control_queue = processing.Queue()
loaded_local_bears_count = len(local_bear_list)
local_bear_list[:], global_bear_list[:] = instantiate_bears(
section,
local_bear_list,
global_bear_list,
complete_file_dict,
message_queue,
console_printer=console_printer,
debug=debug)
loaded_valid_local_bears_count = len(local_bear_list)
# Note: the complete file dict is given as the file dict to bears and
# the whole project is accessible to every bear. However, local bears are
# run only for the changed files if caching is enabled.
# Start tracking all the files
if cache and (loaded_valid_local_bears_count == loaded_local_bears_count
and not use_raw_files):
cache.track_files(set(complete_filename_list))
changed_files = cache.get_uncached_files(
set(filename_list)) if cache else filename_list
# If caching is enabled then the local bears should process only the
# changed files.
logging.debug("coala is run only on changed files, bears' log "
'messages from previous runs may not appear. You may '
'use the `--flush-cache` flag to see them.')
filename_list = changed_files
# Note: the complete file dict is given as the file dict to bears and
# the whole project is accessible to every bear. However, local bears are
# run only for the changed files if caching is enabled.
file_dict = {filename: complete_file_dict[filename]
for filename in filename_list
if filename in complete_file_dict}
bear_runner_args = {'file_name_queue': filename_queue,
'local_bear_list': local_bear_list,
'global_bear_list': global_bear_list,
'global_bear_queue': global_bear_queue,
'file_dict': file_dict,
'local_result_dict': local_result_dict,
'global_result_dict': global_result_dict,
'message_queue': message_queue,
'control_queue': control_queue,
'timeout': 0.1,
'debug': debug}
fill_queue(filename_queue, file_dict.keys())
fill_queue(global_bear_queue, range(len(global_bear_list)))
return ([processing.Process(target=run, kwargs=bear_runner_args)
for i in range(job_count)],
bear_runner_args)
def test_file_invalid(self):
self.assertEqual(collect_files(["invalid_path"], self.log_printer), [])
self.assertEqual([log.message for log in self.log_printer.logs],
["No files matching 'invalid_path' were found."])
def instantiate_processes(section, local_bear_list, global_bear_list,
job_count, cache, log_printer):
"""
Instantiate the number of processes that will run bears which will be
responsible for running bears in a multiprocessing environment.
:param section: The section the bears belong to.
:param local_bear_list: List of local bears belonging to the section.
:param global_bear_list: List of global bears belonging to the section.
:param job_count: Max number of processes to create.
:param cache: An instance of ``misc.Caching.FileCache`` to use as
a file cache buffer.
:param log_printer: The log printer to warn to.
:return: A tuple containing a list of processes,
and the arguments passed to each process which are
the same for each object.
"""
filename_list = collect_files(
glob_list(section.get('files', "")),
log_printer,
ignored_file_paths=glob_list(section.get('ignore', "")),
limit_file_paths=glob_list(section.get('limit_files', "")))
# This stores all matched files irrespective of whether coala is run
# only on changed files or not. Global bears require all the files
complete_filename_list = filename_list
# Start tracking all the files
if cache:
cache.track_files(set(complete_filename_list))
changed_files = cache.get_uncached_files(
set(filename_list)) if cache else filename_list
# If caching is enabled then the local bears should process only the
# changed files.
log_printer.debug(
"coala is run only on changed files, bears' log "
"messages from previous runs may not appear. You may "
"use the `--flush-cache` flag to see them.")
filename_list = changed_files
# Note: the complete file dict is given as the file dict to bears and
# the whole project is accessible to every bear. However, local bears are
# run only for the changed files if caching is enabled.
complete_file_dict = get_file_dict(complete_filename_list, log_printer)
file_dict = {
filename: complete_file_dict[filename]
for filename in filename_list if filename in complete_file_dict
}
manager = multiprocessing.Manager()
global_bear_queue = multiprocessing.Queue()
filename_queue = multiprocessing.Queue()
local_result_dict = manager.dict()
global_result_dict = manager.dict()
message_queue = multiprocessing.Queue()
control_queue = multiprocessing.Queue()
bear_runner_args = {
"file_name_queue": filename_queue,
"local_bear_list": local_bear_list,
"global_bear_list": global_bear_list,
"global_bear_queue": global_bear_queue,
"file_dict": file_dict,
"local_result_dict": local_result_dict,
"global_result_dict": global_result_dict,
"message_queue": message_queue,
"control_queue": control_queue,
"timeout": 0.1
}
local_bear_list[:], global_bear_list[:] = instantiate_bears(
section, local_bear_list, global_bear_list, complete_file_dict,
message_queue)
fill_queue(filename_queue, file_dict.keys())
fill_queue(global_bear_queue, range(len(global_bear_list)))
return ([
multiprocessing.Process(target=run, kwargs=bear_runner_args)
for i in range(job_count)
], bear_runner_args)
def get_project_files(log_printer,
printer,
project_dir,
file_path_completer,
non_interactive=False):
"""
Gets the list of files matching files in the user's project directory
after prompting for glob expressions.
:param log_printer:
A ``LogPrinter`` object.
:param printer:
A ``ConsolePrinter`` object.
:param file_path_completer:
A ``file_path_completer`` object.
:param non_interactive
Whether coala-quickstart is in non-interactive mode
:return:
A list of file paths matching the files.
"""
file_globs = ['**']
ignore_globs = None
gitignore_dir_list = []
for dir_name, subdir_name, file_list in os.walk(project_dir):
if os.path.isfile(os.path.join(dir_name, '.gitignore')):
gitignore_dir_list += [dir_name]
if gitignore_dir_list:
printer.print('The contents of your .gitignore file for the project '
'will be automatically loaded as the files to ignore.',
color='green')
ignore_globs = get_gitignore_glob(project_dir, gitignore_dir_list)
if non_interactive and not ignore_globs:
ignore_globs = []
if ignore_globs is None:
printer.print(GLOB_HELP)
file_path_completer.activate(seed_dir=project_dir)
ignore_globs = ask_question(
'Which files do you want coala to ignore inside the '
'project directory?',
printer=printer,
typecast=list)
file_path_completer.deactivate()
printer.print()
ignore_globs = list(ignore_globs)
escaped_project_dir = glob_escape(project_dir)
file_path_globs = [os.path.join(
escaped_project_dir, glob_exp) for glob_exp in file_globs]
ignore_path_globs = [os.path.join(
escaped_project_dir, glob_exp) for glob_exp in ignore_globs]
ignore_path_globs.append(os.path.join(escaped_project_dir, '.git/**'))
file_paths = collect_files(
file_path_globs,
log_printer,
ignored_file_paths=ignore_path_globs)
return file_paths, ignore_globs
def instantiate_processes(section,
local_bear_list,
global_bear_list,
job_count,
cache,
log_printer,
console_printer):
"""
Instantiate the number of processes that will run bears which will be
responsible for running bears in a multiprocessing environment.
:param section: The section the bears belong to.
:param local_bear_list: List of local bears belonging to the section.
:param global_bear_list: List of global bears belonging to the section.
:param job_count: Max number of processes to create.
:param cache: An instance of ``misc.Caching.FileCache`` to use as
a file cache buffer.
:param log_printer: The log printer to warn to.
:param console_printer: Object to print messages on the console.
:return: A tuple containing a list of processes,
and the arguments passed to each process which are
the same for each object.
"""
filename_list = collect_files(
glob_list(section.get('files', '')),
log_printer,
ignored_file_paths=glob_list(section.get('ignore', '')),
limit_file_paths=glob_list(section.get('limit_files', '')))
# This stores all matched files irrespective of whether coala is run
# only on changed files or not. Global bears require all the files
complete_filename_list = filename_list
# Start tracking all the files
if cache:
cache.track_files(set(complete_filename_list))
changed_files = cache.get_uncached_files(
set(filename_list)) if cache else filename_list
# If caching is enabled then the local bears should process only the
# changed files.
log_printer.debug("coala is run only on changed files, bears' log "
'messages from previous runs may not appear. You may '
'use the `--flush-cache` flag to see them.')
filename_list = changed_files
# Note: the complete file dict is given as the file dict to bears and
# the whole project is accessible to every bear. However, local bears are
# run only for the changed files if caching is enabled.
complete_file_dict = get_file_dict(complete_filename_list, log_printer)
file_dict = {filename: complete_file_dict[filename]
for filename in filename_list
if filename in complete_file_dict}
manager = multiprocessing.Manager()
global_bear_queue = multiprocessing.Queue()
filename_queue = multiprocessing.Queue()
local_result_dict = manager.dict()
global_result_dict = manager.dict()
message_queue = multiprocessing.Queue()
control_queue = multiprocessing.Queue()
bear_runner_args = {'file_name_queue': filename_queue,
'local_bear_list': local_bear_list,
'global_bear_list': global_bear_list,
'global_bear_queue': global_bear_queue,
'file_dict': file_dict,
'local_result_dict': local_result_dict,
'global_result_dict': global_result_dict,
'message_queue': message_queue,
'control_queue': control_queue,
'timeout': 0.1}
local_bear_list[:], global_bear_list[:] = instantiate_bears(
section,
local_bear_list,
global_bear_list,
complete_file_dict,
message_queue,
console_printer=console_printer)
fill_queue(filename_queue, file_dict.keys())
fill_queue(global_bear_queue, range(len(global_bear_list)))
return ([multiprocessing.Process(target=run, kwargs=bear_runner_args)
for i in range(job_count)],
bear_runner_args)
def instantiate_processes(section,
local_bear_list,
global_bear_list,
job_count,
cache,
log_printer,
console_printer,
debug=False,
use_raw_files=False):
"""
Instantiate the number of processes that will run bears which will be
responsible for running bears in a multiprocessing environment.
:param section: The section the bears belong to.
:param local_bear_list: List of local bears belonging to the section.
:param global_bear_list: List of global bears belonging to the section.
:param job_count: Max number of processes to create.
:param cache: An instance of ``misc.Caching.FileCache`` to use as
a file cache buffer.
:param log_printer: The log printer to warn to.
:param use_raw_files: Allow the usage of raw files (non text files)
:param console_printer: Object to print messages on the console.
:param debug: Bypass multiprocessing and activate debug mode
for bears, not catching any exceptions on running
them.
:return: A tuple containing a list of processes,
and the arguments passed to each process which are
the same for each object.
"""
filename_list = collect_files(
glob_list(section.get('files', '')),
log_printer,
ignored_file_paths=glob_list(section.get('ignore', '')),
limit_file_paths=glob_list(section.get('limit_files', '')),
section_name=section.name)
# This stores all matched files irrespective of whether coala is run
# only on changed files or not. Global bears require all the files
complete_filename_list = filename_list
complete_file_dict = get_file_dict(complete_filename_list, log_printer,
use_raw_files)
if debug:
from . import DebugProcessing as processing
else:
import multiprocessing as processing
manager = processing.Manager()
global_bear_queue = processing.Queue()
filename_queue = processing.Queue()
local_result_dict = manager.dict()
global_result_dict = manager.dict()
message_queue = processing.Queue()
control_queue = processing.Queue()
loaded_local_bears_count = len(local_bear_list)
local_bear_list[:], global_bear_list[:] = instantiate_bears(
section,
local_bear_list,
global_bear_list,
complete_file_dict,
message_queue,
console_printer=console_printer,
debug=debug)
loaded_valid_local_bears_count = len(local_bear_list)
# Note: the complete file dict is given as the file dict to bears and
# the whole project is accessible to every bear. However, local bears are
# run only for the changed files if caching is enabled.
# Start tracking all the files
if cache and (loaded_valid_local_bears_count == loaded_local_bears_count
and not use_raw_files):
cache.track_files(set(complete_filename_list))
changed_files = cache.get_uncached_files(
set(filename_list)) if cache else filename_list
# If caching is enabled then the local bears should process only the
# changed files.
log_printer.debug("coala is run only on changed files, bears' log "
'messages from previous runs may not appear. You may '
'use the `--flush-cache` flag to see them.')
filename_list = changed_files
# Note: the complete file dict is given as the file dict to bears and
# the whole project is accessible to every bear. However, local bears are
# run only for the changed files if caching is enabled.
file_dict = {filename: complete_file_dict[filename]
for filename in filename_list
if filename in complete_file_dict}
bear_runner_args = {'file_name_queue': filename_queue,
'local_bear_list': local_bear_list,
'global_bear_list': global_bear_list,
'global_bear_queue': global_bear_queue,
'file_dict': file_dict,
'local_result_dict': local_result_dict,
'global_result_dict': global_result_dict,
'message_queue': message_queue,
'control_queue': control_queue,
'timeout': 0.1,
'debug': debug}
fill_queue(filename_queue, file_dict.keys())
fill_queue(global_bear_queue, range(len(global_bear_list)))
return ([processing.Process(target=run, kwargs=bear_runner_args)
for i in range(job_count)],
bear_runner_args)
def test_file_invalid(self):
self.assertEqual(collect_files(["invalid_path"]), [])
def test_get_project_files_gitignore(self):
orig_cwd = os.getcwd()
os.chdir(os.path.dirname(os.path.realpath(__file__)))
os.makedirs("file_globs_gitignore_testfiles", exist_ok=True)
os.chdir("file_globs_gitignore_testfiles")
with open(".gitignore", "w") as f:
f.write("""
# Start of gitignore
build
ignore.c
/tests
/upload.c
/*.py
*.pyc
__pycache__
# End of gitignore""")
os.makedirs("another_folder", exist_ok=True)
os.chdir("another_folder")
with open('.gitignore', 'w') as f:
f.write("""
# Start of gitignore
*.js
# End of gitignore""")
os.chdir(os.path.dirname(os.path.realpath(__file__)))
os.chdir("file_globs_gitignore_testfiles")
os.makedirs("data", exist_ok=True)
os.chdir("data")
os.makedirs("sample", exist_ok=True)
os.chdir("sample")
with open('.gitignore', 'w') as f:
f.write("""
# Start of gitignore
*.html
# End of gitignore""")
os.chdir(os.path.dirname(os.path.realpath(__file__)))
os.chdir("file_globs_gitignore_testfiles")
files = [
os.path.join("src", "main.c"),
os.path.join("src", "main.h"),
os.path.join("src", "lib", "ssl.c"),
os.path.join("src", "tests", "main.c"),
os.path.join("src", "main.py"),
os.path.join("src", "upload.c"),
os.path.join("another_folder", "another_file.c"),
os.path.join("data", "sample", "index.css"),
os.path.join("data", "example.py"), ".coafile"
]
ignored_files = [
os.path.join("build", "main.c"),
os.path.join("tests", "run.c"),
os.path.join("src", "build", "main.c"), "ignore.c",
os.path.join("src", "ignore.c"), "globexp.py", "upload.c",
os.path.join("src", "main.pyc"),
os.path.join("another_folder", "script.js"),
os.path.join("data", "sample", "index.html"), "run.pyc"
]
for file in files + ignored_files:
os.makedirs(os.path.dirname(os.path.abspath(file)), exist_ok=True)
open(file, "w").close()
files += [".gitignore"]
files += [os.path.join("another_folder", ".gitignore")]
files += [os.path.join("data", "sample", ".gitignore")]
gitignore_dir_list = [
os.getcwd(),
os.path.join(os.getcwd(), "another_folder"),
os.path.join(os.getcwd(), "data", "sample")
]
globs = list(get_gitignore_glob(os.getcwd(), gitignore_dir_list))
returned_files = collect_files([os.path.join(os.getcwd(), "**")],
self.log_printer,
ignored_file_paths=globs)
files = [os.path.normcase(os.path.abspath(file)) for file in files]
ignored_files = [os.path.abspath(file) for file in ignored_files]
self.maxDiff = None
self.assertEqual(sorted(files), sorted(returned_files))
with suppress_stdout():
self.assertEqual(
sorted(
get_project_files(self.log_printer, self.printer,
os.getcwd(),
self.file_path_completer)[0]),
sorted(files))
os.remove(os.path.join("another_folder", ".gitignore"))
os.remove(os.path.join("data", "sample", ".gitignore"))
os.remove(".gitignore")
os.chdir(orig_cwd)
def test_get_project_files_gitignore(self):
orig_cwd = os.getcwd()
os.chdir(os.path.dirname(os.path.realpath(__file__)))
os.makedirs("file_globs_gitignore_testfiles", exist_ok=True)
os.chdir("file_globs_gitignore_testfiles")
with open(".gitignore", "w") as f:
f.write("""
# Start of gitignore
build
ignore.c
/tests
/upload.c
/*.py
*.pyc
__pycache__
# End of gitignore""")
os.makedirs("another_folder", exist_ok=True)
os.chdir("another_folder")
with open('.gitignore', 'w') as f:
f.write("""
# Start of gitignore
*.js
# End of gitignore""")
os.chdir(os.path.dirname(os.path.realpath(__file__)))
os.chdir("file_globs_gitignore_testfiles")
os.makedirs("data", exist_ok=True)
os.chdir("data")
os.makedirs("sample", exist_ok=True)
os.chdir("sample")
with open('.gitignore', 'w') as f:
f.write("""
# Start of gitignore
*.html
# End of gitignore""")
os.chdir(os.path.dirname(os.path.realpath(__file__)))
os.chdir("file_globs_gitignore_testfiles")
files = [os.path.join("src", "main.c"),
os.path.join("src", "main.h"),
os.path.join("src", "lib", "ssl.c"),
os.path.join("src", "tests", "main.c"),
os.path.join("src", "main.py"),
os.path.join("src", "upload.c"),
os.path.join("another_folder", "another_file.c"),
os.path.join("data", "sample", "index.css"),
os.path.join("data", "example.py"),
".coafile"]
ignored_files = [os.path.join("build", "main.c"),
os.path.join("tests", "run.c"),
os.path.join("src", "build", "main.c"),
"ignore.c",
os.path.join("src", "ignore.c"),
"globexp.py",
"upload.c",
os.path.join("src", "main.pyc"),
os.path.join("another_folder", "script.js"),
os.path.join("data", "sample", "index.html"),
"run.pyc"]
for file in files + ignored_files:
os.makedirs(os.path.dirname(os.path.abspath(file)), exist_ok=True)
open(file, "w").close()
files += [".gitignore"]
files += [os.path.join("another_folder", ".gitignore")]
files += [os.path.join("data", "sample", ".gitignore")]
gitignore_dir_list = [os.getcwd(),
os.path.join(os.getcwd(), "another_folder"),
os.path.join(os.getcwd(), "data", "sample")]
globs = list(get_gitignore_glob(os.getcwd(), gitignore_dir_list))
returned_files = collect_files(
[os.path.join(os.getcwd(), "**")],
self.log_printer,
ignored_file_paths=globs)
files = [os.path.normcase(os.path.abspath(file)) for file in files]
ignored_files = [os.path.abspath(file) for file in ignored_files]
self.maxDiff = None
self.assertEqual(sorted(files), sorted(returned_files))
with suppress_stdout():
self.assertEqual(
sorted(get_project_files(self.log_printer,
self.printer,
os.getcwd(),
self.file_path_completer)[0]),
sorted(files))
os.remove(os.path.join("another_folder", ".gitignore"))
os.remove(os.path.join("data", "sample", ".gitignore"))
os.remove(".gitignore")
os.chdir(orig_cwd)
