def main(argv):
"""Main entry point."""
if len(argv) > 1:
raise app.UsageError("Unknown arguments '{}'".format(', '.join(
argv[1:])))
clone_list_path = pathlib.Path(FLAGS.clone_list or "")
if not clone_list_path.is_file():
raise app.UsageError('--clone_list is not a file.')
clone_list = pbutil.FromFile(clone_list_path,
scrape_repos_pb2.LanguageCloneList())
# Error early if the config contains invalid preprocessors.
for language in clone_list.language:
for importer in language.importer:
[
preprocessors.GetPreprocessorFunction(p)
for p in importer.preprocessor
]
pool = multiprocessing.Pool(FLAGS.processes)
for language in clone_list.language:
d = pathlib.Path(language.destination_directory)
d = d.parent / (str(d.name) + '.db')
db = contentfiles.ContentFiles(d)
if pathlib.Path(language.destination_directory).is_dir():
ImportFromLanguage(db, language, pool)
def main(argv):
"""Main entry point."""
if len(argv) > 1:
raise app.UsageError("Unknown arguments '{}'".format(", ".join(argv[1:])))
clone_list_path = pathlib.Path(FLAGS.clone_list or "")
if not clone_list_path.is_file():
raise app.UsageError("--clone_list is not a file.")
clone_list = pbutil.FromFile(
clone_list_path, scrape_repos_pb2.LanguageCloneList()
)
if not FLAGS.export_path:
raise app.UsageError("--export_path not set.")
export_path = pathlib.Path(FLAGS.export_path)
export_path.mkdir(parents=True, exist_ok=True)
# To export from contentfiles database.
for language in clone_list.language:
d = pathlib.Path(language.destination_directory)
d = d.parent / (str(d.name) + ".db")
db = contentfiles.ContentFiles(f"sqlite:///{d}")
with db.Session() as session:
(export_path / language.language).mkdir(exist_ok=True)
ExportDatabase(session, export_path / language.language)
def main(argv):
"""Main entry point."""
if len(argv) > 1:
raise app.UsageError("Unknown arguments: '{}'.".format(" ".join(argv[1:])))
language = FLAGS.lang
if not language in LANGUAGE_TO_CLANG_ARGS:
raise app.UsageError(
f"Language `{language}` not supported. "
f"Must be one of: {LANGUAGE_TO_CLANG_ARGS.keys()}"
)
db = bytecode_database.Database(FLAGS.db)
cf = contentfiles.ContentFiles(FLAGS.cf)
PopulateBytecodeTable(cf, language, db)
def db(tempdir: pathlib.Path) -> contentfiles.ContentFiles:
db_ = contentfiles.ContentFiles(f"sqlite:///{tempdir}/a")
with db_.Session(commit=True) as session:
session.add(
contentfiles.GitHubRepository(
owner="foo",
name="bar",
clone_from_url="abc",
num_stars=0,
num_forks=0,
num_watchers=0,
active=1,
exported=0,
date_scraped=datetime.datetime.utcnow(),
language="java",
)
)
session.add(
contentfiles.ContentFile(
clone_from_url="abc",
relpath="foo",
artifact_index=0,
sha256="000",
charcount=100,
linecount=4,
active=1,
text="""
import java.util.ArrayList;
public class HelloWorld {
private int foo(ArrayList<Integer> x) {
return 5;
}
public static void main(String[] args) {
System.out.println("Hello, world");
}
}
""",
)
)
return db_
def test_db(tempdir) -> contentfiles.ContentFiles:
yield contentfiles.ContentFiles(tempdir / 'test.db')
def test_db(tempdir) -> contentfiles.ContentFiles:
yield contentfiles.ContentFiles(f'sqlite:///{tempdir}/test.db')
def db(tempdir: pathlib.Path) -> contentfiles.ContentFiles:
return contentfiles.ContentFiles(f"sqlite:///{tempdir}/db")