def generate_settings(project_dir,
project_files,
ignore_globs,
relevant_bears,
incomplete_sections=False):
"""
Generates the settings for the given project.
:param project_dir:
Full path of the user's project directory.
:param project_files:
A list of file paths matched in the user's project directory.
:param ignore_globs:
The list of ignore glob expressions.
:param relevant_bears:
A dict with language name as key and bear classes as value.
:param incomplete_sections:
When bears with non optional settings are found, user is asked for
setting value of non optional bears and then a ``Section`` object
having ``files`` field, ``bears`` field and settings, is returned.
If incomplete_sections is set to ``True``, then no user input will
be asked and a ``Section`` object having only the ``files`` and
``bears`` field will be returned.
In CI mode, bears with non optional setting are not added in coafile.
But if incomplete_sections is set to ``True`` in CI mode, then those
bears are also added in the coafile.
:return:
A dict with section name as key and a ``Section`` object as value.
"""
lang_map = {lang.lower(): lang for lang in relevant_bears}
lang_files = split_by_language(project_files)
extset = get_extensions(project_files)
settings = OrderedDict()
settings["default"] = generate_section(
"default", [ext for lang in lang_files for ext in extset[lang]],
relevant_bears[lang_map["all"]])
ignored_files = generate_ignore_field(project_dir, lang_files.keys(),
extset, ignore_globs)
if ignored_files:
settings["default"]["ignore"] = ignored_files
for lang in lang_files:
if lang != "unknown" and lang != "all":
settings[lang_map[lang]] = generate_section(
lang, extset[lang], relevant_bears[lang_map[lang]])
log_printer = LogPrinter(ConsolePrinter())
if not incomplete_sections:
fill_settings(settings, acquire_settings, log_printer)
return settings
def test_split_by_language(self):
with NamedTemporaryFile(delete=False, suffix='.py') as temp_file1, \
NamedTemporaryFile(delete=False, suffix='.txt') as temp_file2, \
NamedTemporaryFile(delete=False, suffix='.txt') as temp_file3:
temp_file3.write(b'#!bin/python')
temp_file3.close()
langs = split_by_language(
[temp_file1.name, temp_file2.name, temp_file3.name])
self.assertCountEqual(
langs, {
'all': [temp_file1.name, temp_file3.name],
'python': [temp_file1.name, temp_file3.name],
})
def generate_settings(project_dir, project_files, ignore_globs, relevant_bears):
"""
Generates the settings for the given project.
:param project_dir:
Full path of the user's project directory.
:param project_files:
A list of file paths matched in the user's project directory.
:param ignore_globs:
The list of ignore glob expressions.
:param relevant_bears:
A dict with language name as key and bear classes as value.
:return:
A dict with section name as key and a ``Section`` object as value.
"""
lang_map = {lang.lower(): lang for lang in relevant_bears}
lang_files = split_by_language(project_files)
extset = get_extensions(project_files)
settings = OrderedDict()
settings["default"] = generate_section(
"default",
[ext for lang in lang_files for ext in extset[lang]],
relevant_bears[lang_map["all"]])
settings["default"]["ignore"] = generate_ignore_field(
project_dir,
lang_files.keys(),
extset,
ignore_globs)
for lang in lang_files:
if lang != "unknown" and lang != "all":
settings[lang_map[lang]] = generate_section(
lang,
extset[lang],
relevant_bears[lang_map[lang]])
log_printer = LogPrinter(ConsolePrinter())
fill_settings(settings, acquire_settings, log_printer)
return settings
def local_bear_test(bear, file_dict, file_names, lang, kwargs,
ignore_ranges):
lang_files = split_by_language(file_names)
lang_files = {k.lower(): v for k, v in lang_files.items()}
import multiprocessing as mp
pool = mp.Pool(processes=mp.cpu_count()-1)
file_results = []
for file in lang_files[lang.lower()]:
kwargs['filename'] = [file]
kwargs['file'] = [file_dict[file]]
results = []
values = []
for vals in itertools.product(*kwargs.values()):
print_val = dict(zip(kwargs, vals))
print_val.pop('file', None)
values.append(vals)
section = Section('test-section-local-bear')
bear_obj = bear(section, None)
ret_val = bear_obj.run(**dict(zip(kwargs, vals)))
ret_val = list(ret_val)
# FIXME: Multiprocessing not working on windows.
if os.name == 'nt': # pragma posix: no cover
results.append(check_bear_results(ret_val, ignore_ranges))
else: # pragma nt: no cover
results.append(pool.apply(check_bear_results,
args=(ret_val, ignore_ranges)))
for index, result in enumerate(results):
if result is True:
# A set of bear setting values is found to be green
# for a particular file
arguments = dict(zip(kwargs, values[index]))
arguments.pop('file')
file_results.append(arguments)
return {bear: file_results}
def generate_settings(project_dir,
project_files,
ignore_globs,
relevant_bears,
extracted_info,
incomplete_sections=False,
log_printer=None):
"""
Generates the settings for the given project.
:param project_dir:
Full path of the user's project directory.
:param project_files:
A list of file paths matched in the user's project directory.
:param ignore_globs:
The list of ignore glob expressions.
:param relevant_bears:
A dict with language name as key and bear classes as value.
:param extracted_info:
A list information extracted from the project files by
``InfoExtractor`` classes.
:param incomplete_sections:
When bears with non optional settings are found, user is asked for
setting value of non optional bears and then a ``Section`` object
having ``files`` field, ``bears`` field and settings, is returned.
If incomplete_sections is set to ``True``, then no user input will
be asked and a ``Section`` object having only the ``files`` and
``bears`` field will be returned.
In CI mode, bears with non optional setting are not added in coafile.
But if incomplete_sections is set to ``True`` in CI mode, then those
bears are also added in the coafile.
:return:
A dict with section name as key and a ``Section`` object as value.
"""
lang_map = {lang.lower(): lang for lang in relevant_bears}
lang_files = split_by_language(project_files)
extset = get_extensions(project_files)
settings = OrderedDict()
settings['all'] = generate_section(
'all', [ext for lang in lang_files for ext in extset[lang]],
relevant_bears[lang_map['all']])
ignored_files = generate_ignore_field(project_dir, lang_files.keys(),
extset, ignore_globs)
if ignored_files:
settings['all']['ignore'] = ignored_files
for lang in lang_files:
if lang != 'unknown' and lang != 'all':
settings['all.' + lang_map[lang]] = generate_section(
'all.' + lang, extset[lang], relevant_bears[lang_map[lang]])
if not incomplete_sections:
fill_settings(settings,
acquire_settings,
log_printer,
fill_section_method=fill_section,
extracted_info=extracted_info)
return settings
def generate_green_mode_sections(data,
project_dir,
project_files,
ignore_globs,
printer=None,
suffix=''):
"""
Generates the section objects for the green_mode.
:param data:
This is the data structure generated from the method
generate_data_struct_for_sections().
:param project_dir:
The path of the project directory.
:param project_files:
List of paths to only the files inside the project directory.
:param ignore_globs:
The globs of files to ignore.
:param printer:
The ConsolePrinter object.
:param suffix:
A suffix that can be added to the `.coafile.green`.
"""
all_sections = {
'all': [],
}
lang_files = split_by_language(project_files)
extset = get_extensions(project_files)
ignored_files = generate_ignore_field(project_dir, lang_files.keys(),
extset, ignore_globs)
if ignored_files:
section_all = Section('all')
section_all['ignore'] = ignored_files
all_sections['all'].append(section_all)
for bear in data:
num = 0
bear_sections = []
for setting_combs in data[bear]:
if not setting_combs:
continue
for dict_ in setting_combs:
num = num + 1
section = Section('all.' + bear.__name__ + str(num), None)
for key_ in dict_:
if key_ == 'filename':
file_list_sec = dict_[key_]
file_list_sec, ignore_list = aggregate_files(
file_list_sec, project_dir)
dict_[key_] = file_list_sec
section['ignore'] = ', '.join(
escape(x, '\\') for x in ignore_list)
section['files'] = ', '.join(
escape(x, '\\') for x in dict_[key_])
else:
section[key_] = str(dict_[key_])
section['bears'] = bear.__name__
bear_sections.append(section)
# Remove sections for the same bear who have common files i.e. more
# than one setting was found to be green.
file_list = []
new_bear_sections = []
for index, section in enumerate(bear_sections):
if not section.contents['files'] in file_list:
file_list.append(section.contents['files'])
new_bear_sections.append(section)
all_sections[bear.__name__] = new_bear_sections
coafile = os.path.join(project_dir, '.coafile.green' + suffix)
writer = ConfWriter(coafile)
write_sections(writer, all_sections)
writer.close()
printer.print("'" + coafile + "' successfully generated.", color='green')
def local_bear_test(
bear,
file_dict,
file_names,
lang,
kwargs,
ignore_ranges,
jobs: int = 0,
):
lang_files = split_by_language(file_names)
lang_files = {k.lower(): v for k, v in lang_files.items()}
pool = _create_mp_pool(jobs)
file_results = []
for file in lang_files[lang.lower()]:
kwargs['filename'] = [file]
kwargs['file'] = [file_dict[file]]
results = []
values = []
for vals in itertools.product(*kwargs.values()):
flag = 0
for dep in bear.BEAR_DEPS:
section = Section('dep-bear')
bear_obj = dep(section, None)
arguments = dict(zip(kwargs, vals))
dep_args = get_all_args(dep.run)
new_arguments = {}
for arg_ in arguments.keys():
if arg_ in dep_args.keys(): # pragma: no cover
new_arguments[arg_] = arguments[arg_]
arguments = new_arguments
ret_val = bear_obj.run(**arguments)
ret_val = [] if not ret_val else list(ret_val)
dep_res = check_bear_results(ret_val, ignore_ranges)
if not dep_res:
flag = 1
if flag == 1:
continue
print_val = dict(zip(kwargs, vals))
print_val.pop('file', None)
values.append(vals)
section = Section('test-section-local-bear')
bear_obj = bear(section, None)
ret_val = bear_obj.run(**dict(zip(kwargs, vals)))
ret_val = [] if not ret_val else list(ret_val)
if pool: # pragma Python 3.5: no cover; pragma nt: no cover
results.append(
pool.apply(check_bear_results,
args=(ret_val, ignore_ranges)))
else: # pragma Python 3.4,3.6,3.7: no cover
results.append(check_bear_results(ret_val, ignore_ranges))
for index, result in enumerate(results):
if result is True:
# A set of bear setting values is found to be green
# for a particular file
arguments = dict(zip(kwargs, values[index]))
arguments.pop('file')
file_results.append(arguments)
return {bear: file_results}
def generate_green_mode_sections(data, project_dir, project_files,
ignore_globs, printer=None, suffix=''):
"""
Generates the section objects for the green_mode.
:param data:
This is the data structure generated from the method
generate_data_struct_for_sections().
:param project_dir:
The path of the project directory.
:param project_files:
List of paths to only the files inside the project directory.
:param ignore_globs:
The globs of files to ignore.
:param printer:
The ConsolePrinter object.
:param suffix:
A suffix that can be added to the `.coafile.green`.
"""
all_sections = {'all': [], }
lang_files = split_by_language(project_files)
extset = get_extensions(project_files)
ignored_files = generate_ignore_field(project_dir, lang_files.keys(),
extset, ignore_globs)
if ignored_files:
section_all = Section('all')
section_all['ignore'] = ignored_files
all_sections['all'].append(section_all)
for bear in data:
num = 0
bear_sections = []
for setting_combs in data[bear]:
if not setting_combs:
continue
for dict_ in setting_combs:
num = num + 1
section = Section('all.' + bear.__name__ + str(num), None)
for key_ in dict_:
if key_ == 'filename':
file_list_sec = dict_[key_]
file_list_sec, ignore_list = aggregate_files(
file_list_sec, project_dir)
dict_[key_] = file_list_sec
section['ignore'] = ', '.join(
escape(x, '\\') for x in ignore_list)
section['files'] = ', '.join(
escape(x, '\\') for x in dict_[key_])
else:
section[key_] = str(dict_[key_])
section['bears'] = bear.__name__
bear_sections.append(section)
# Remove sections for the same bear who have common files i.e. more
# than one setting was found to be green.
file_list = []
new_bear_sections = []
for index, section in enumerate(bear_sections):
if not section.contents['files'] in file_list:
file_list.append(section.contents['files'])
new_bear_sections.append(section)
all_sections[bear.__name__] = new_bear_sections
coafile = os.path.join(project_dir, '.coafile.green' + suffix)
writer = ConfWriter(coafile)
write_sections(writer, all_sections)
writer.close()
printer.print("'" + coafile + "' successfully generated.", color='green')
def local_bear_test(bear, file_dict, file_names, lang, kwargs,
ignore_ranges,
jobs: int = 0,
):
lang_files = split_by_language(file_names)
lang_files = {k.lower(): v for k, v in lang_files.items()}
pool = _create_mp_pool(jobs)
file_results = []
for file in lang_files[lang.lower()]:
kwargs['filename'] = [file]
kwargs['file'] = [file_dict[file]]
results = []
values = []
for vals in itertools.product(*kwargs.values()):
flag = 0
for dep in bear.BEAR_DEPS:
section = Section('dep-bear')
bear_obj = dep(section, None)
arguments = dict(zip(kwargs, vals))
dep_args = get_all_args(dep.run)
new_arguments = {}
for arg_ in arguments.keys():
if arg_ in dep_args.keys(): # pragma: no cover
new_arguments[arg_] = arguments[arg_]
arguments = new_arguments
ret_val = bear_obj.run(**arguments)
ret_val = [] if not ret_val else list(ret_val)
dep_res = check_bear_results(ret_val, ignore_ranges)
if not dep_res:
flag = 1
if flag == 1:
continue
print_val = dict(zip(kwargs, vals))
print_val.pop('file', None)
values.append(vals)
section = Section('test-section-local-bear')
bear_obj = bear(section, None)
ret_val = bear_obj.run(**dict(zip(kwargs, vals)))
ret_val = [] if not ret_val else list(ret_val)
if pool: # pragma Python 3.5: no cover; pragma nt: no cover
results.append(pool.apply(check_bear_results,
args=(ret_val, ignore_ranges)))
else: # pragma Python 3.4,3.6,3.7: no cover
results.append(check_bear_results(ret_val, ignore_ranges))
for index, result in enumerate(results):
if result is True:
# A set of bear setting values is found to be green
# for a particular file
arguments = dict(zip(kwargs, values[index]))
arguments.pop('file')
file_results.append(arguments)
return {bear: file_results}