def test_ImportFromLanguage_Java_repo(
test_db: contentfiles.ContentFiles, tempdir: pathlib.Path
):
"""An end-to-end test of a Java importer."""
(tempdir / "Owner_Name" / ".git").mkdir(parents=True)
(tempdir / "Owner_Name" / "src").mkdir(parents=True)
# A repo will only be imported if there is a repo meta file.
pbutil.ToFile(
scrape_repos_pb2.GitHubRepoMetadata(owner="Owner", name="Name"),
tempdir / "Owner_Name.pbtxt",
)
# Create some files in our test repo.
with open(tempdir / "Owner_Name" / "src" / "A.java", "w") as f:
f.write(
"""
public class A {
public static void helloWorld() {
System.out.println("Hello, world!");
}
}
"""
)
with open(tempdir / "Owner_Name" / "src" / "B.java", "w") as f:
f.write(
"""
public class B {
private static int foo() {return 5;}
}
"""
)
with open(tempdir / "Owner_Name" / "README.txt", "w") as f:
f.write("Hello, world!")
language = scrape_repos_pb2.LanguageToClone(
language="foolang",
query=[],
destination_directory=str(tempdir),
importer=[
scrape_repos_pb2.ContentFilesImporterConfig(
source_code_pattern=".*\\.java",
preprocessor=[
"datasets.github.scrape_repos.preprocessors." "extractors:JavaMethods"
],
),
],
)
importer.ImportFromLanguage(test_db, language)
with test_db.Session() as session:
query = session.query(contentfiles.ContentFile)
assert query.count() == 2
assert set([cf.text for cf in query]) == {
(
"public static void helloWorld(){\n"
' System.out.println("Hello, world!");\n}\n'
),
"private static int foo(){\n return 5;\n}\n",
}
def test_ImportFromLanguage_Java_repo(tempdir: pathlib.Path):
"""An end-to-end test of a Java importer."""
(tempdir / 'src').mkdir()
(tempdir / 'src' / 'Owner_Name' / '.git').mkdir(parents=True)
(tempdir / 'src' / 'Owner_Name' / 'src').mkdir(parents=True)
# A repo will only be imported if there is a repo meta file.
pbutil.ToFile(scrape_repos_pb2.GitHubRepoMetadata(
owner='Owner',
name='Name'),
tempdir / 'src' / 'Owner_Name.pbtxt')
# Create some files in our test repo.
with open(tempdir / 'src' / 'Owner_Name' / 'src' / 'A.java', 'w') as f:
f.write("""
public class A {
public static void helloWorld() {
System.out.println("Hello, world!");
}
}
""")
with open(tempdir / 'src' / 'Owner_Name' / 'src' / 'B.java', 'w') as f:
f.write("""
public class B {
private static int foo() {return 5;}
}
""")
with open(tempdir / 'src' / 'Owner_Name' / 'README.txt', 'w') as f:
f.write('Hello, world!')
language = scrape_repos_pb2.LanguageToClone(
language='foolang',
query=[],
destination_directory=str(tempdir / 'src'),
importer=[
scrape_repos_pb2.ContentFilesImporterConfig(
source_code_pattern='.*\\.java',
preprocessor=["datasets.github.scrape_repos.preprocessors."
"extractors:JavaMethods"]),
]
)
indexer.ImportFromLanguage(language, multiprocessing.Pool(1))
test_repo = github_repo.GitHubRepo(tempdir / 'src' / 'Owner_Name.pbtxt')
assert (test_repo.index_dir / 'DONE.txt').is_file()
assert len(list(test_repo.index_dir.iterdir())) == 3
contentfiles = list(test_repo.ContentFiles())
assert len(contentfiles) == 2
assert set([cf.text for cf in contentfiles]) == {
('public static void helloWorld(){\n'
' System.out.println("Hello, world!");\n}\n'),
'private static int foo(){\n return 5;\n}\n',
}
def test_GitHubRepo_Index_not_cloned(test_repo: github_repo.GitHubRepo):
"""Indexing a repo which is not cloned does nothing."""
fs.rm(test_repo.clone_dir)
assert not test_repo.IsIndexed()
test_repo.Index([
scrape_repos_pb2.ContentFilesImporterConfig(
source_code_pattern='.*\\.java',
preprocessor=[
"datasets.github.scrape_repos.preprocessors."
"extractors:JavaMethods"
]),
], multiprocessing.Pool(1))
assert not test_repo.IsIndexed()
def test_GitHubRepo_Index_index_dir_paths(tempdir: pathlib.Path):
"""Test that index directories are produced in the correct location."""
repo = _CreateTestRepo(tempdir / 'java', 'Foo', 'Bar')
repo.Index([
scrape_repos_pb2.ContentFilesImporterConfig(
source_code_pattern='.*\\.java',
preprocessor=[
"datasets.github.scrape_repos.preprocessors."
"extractors:JavaMethods"
]),
], multiprocessing.Pool(1))
assert (tempdir / 'java.index').is_dir()
assert (tempdir / 'java.index' / 'Foo_Bar').is_dir()
def language(
tempdir: pathlib.Path, query: scrape_repos_pb2.GitHubRepositoryQuery
) -> scrape_repos_pb2.LanguageCloneList:
return scrape_repos_pb2.LanguageToClone(
language="java",
query=[query],
destination_directory=str(tempdir),
importer=[
scrape_repos_pb2.ContentFilesImporterConfig(
source_code_pattern=".*\\.java"
)
],
)
def test_GitHubRepo_Index_Java_repo(test_repo: github_repo.GitHubRepo):
"""An end-to-end test of a Java indexer."""
(test_repo.clone_dir / "src").mkdir(exist_ok=True)
with open(test_repo.clone_dir / "src" / "A.java", "w") as f:
f.write("""
public class A {
public static void helloWorld() {
System.out.println("Hello, world!");
}
}
""")
with open(test_repo.clone_dir / "src" / "B.java", "w") as f:
f.write("""
public class B {
private static int foo() {return 5;}
}
""")
with open(test_repo.clone_dir / "README.txt", "w") as f:
f.write("Hello, world!")
assert not test_repo.index_dir.is_dir()
assert not list(test_repo.ContentFiles())
test_repo.Index(
[
scrape_repos_pb2.ContentFilesImporterConfig(
source_code_pattern=".*\\.java",
preprocessor=[
"datasets.github.scrape_repos.preprocessors."
"extractors:JavaMethods"
],
),
],
multiprocessing.Pool(1),
)
assert test_repo.index_dir.is_dir()
assert (test_repo.index_dir / "DONE.txt").is_file()
assert len(list(test_repo.index_dir.iterdir())) == 3
contentfiles = list(test_repo.ContentFiles())
assert len(contentfiles) == 2
assert set([cf.text for cf in contentfiles]) == {
("public static void helloWorld(){\n"
' System.out.println("Hello, world!");\n}\n'),
"private static int foo(){\n return 5;\n}\n",
}
def GetLanguageToClone(
query_prefix: str, destination_dir: str
) -> scrape_repos_pb2.LanguageToClone:
# See: https://help.github.com/en/articles/sorting-search-results
sort_by = random.choice(["stars", "forks", "updated"])
return scrape_repos_pb2.LanguageToClone(
language="java",
query=[
scrape_repos_pb2.GitHubRepositoryQuery(
string=f"{query_prefix} language:java sort:{sort_by} fork:false"
)
],
destination_directory=destination_dir,
importer=[
scrape_repos_pb2.ContentFilesImporterConfig(
source_code_pattern=".*\\.java"
)
],
clone_from_url_blacklist=BLACKLIST_GITHUB_REPOS,
)