def test_GetPreprocessFunction_undecorated_preprocessor():
"""Test that an ValueError is raised if preprocessor not decorated."""
with pytest.raises(ValueError) as e_info:
preprocessors.GetPreprocessorFunction(
'datasets.github.scrape_repos.preprocessors.preprocessors_test'
':MockUndecoratedPreprocessor')
assert '@dataset_preprocessor' in str(e_info.value)
def test_GetPreprocessFunction_missing_function():
"""Test that ValueError is raised if module exists but function doesn't."""
with pytest.raises(ValueError) as e_info:
preprocessors.GetPreprocessorFunction(
'datasets.github.scrape_repos.preprocessors.preprocessors_test:Foo'
)
assert 'not found' in str(e_info.value)
def main(argv):
"""Main entry point."""
if len(argv) > 1:
raise app.UsageError("Unknown arguments '{}'".format(', '.join(
argv[1:])))
clone_list_path = pathlib.Path(FLAGS.clone_list or "")
if not clone_list_path.is_file():
raise app.UsageError('--clone_list is not a file.')
clone_list = pbutil.FromFile(clone_list_path,
scrape_repos_pb2.LanguageCloneList())
# Error early if the config contains invalid preprocessors.
for language in clone_list.language:
for importer in language.importer:
[
preprocessors.GetPreprocessorFunction(p)
for p in importer.preprocessor
]
pool = multiprocessing.Pool(FLAGS.processes)
for language in clone_list.language:
d = pathlib.Path(language.destination_directory)
d = d.parent / (str(d.name) + '.db')
db = contentfiles.ContentFiles(d)
if pathlib.Path(language.destination_directory).is_dir():
ImportFromLanguage(db, language, pool)
def main(argv):
"""Main entry point."""
if len(argv) > 1:
raise app.UsageError("Unknown arguments '{}'".format(", ".join(
argv[1:])))
clone_list_path = pathlib.Path(FLAGS.clone_list or "")
if not clone_list_path.is_file():
raise app.UsageError("--clone_list is not a file.")
clone_list = pbutil.FromFile(clone_list_path,
scrape_repos_pb2.LanguageCloneList())
# Error early if the config contains invalid preprocessors.
for language in clone_list.language:
for importer in language.importer:
[
preprocessors.GetPreprocessorFunction(p)
for p in importer.preprocessor
]
pool = multiprocessing.Pool(FLAGS.indexer_processes)
for language in clone_list.language:
ImportFromLanguage(language, pool)
def test_GetPreprocessFunction_mock_preprocessor():
"""Test that a mock preprocessor can be found."""
f = preprocessors.GetPreprocessorFunction(
'datasets.github.scrape_repos.preprocessors.preprocessors_test:MockPreprocessor'
)
assert f == MockPreprocessor
def test_GetPreprocessFunction_missing_module():
"""Test that ValueError is raised if module not found."""
with pytest.raises(ValueError) as e_info:
preprocessors.GetPreprocessorFunction('not.a.real.module:Foo')
assert 'not found' in str(e_info.value)
def test_GetPreprocessFunction_empty_string():
"""Test that a ValueError is raised if no preprocessor is given."""
with pytest.raises(ValueError) as e_info:
preprocessors.GetPreprocessorFunction('')
assert 'Invalid preprocessor name' in str(e_info.value)
