def prepare(self):
self.db.add_filler(
generic.SourcesFiller(source=[
'GitHub',
],
source_urlroot=[
'github.com',
]))
if os.path.isabs(self.packages_file):
self.db.add_filler(
generic.PackageFiller(
package_list_file=os.path.basename(self.packages_file),
data_folder=os.path.dirname(self.packages_file)))
else:
self.db.add_filler(
generic.PackageFiller(package_list_file=self.packages_file))
self.db.add_filler(generic.RepositoriesFiller())
if os.path.isabs(self.api_keys_file):
data_folder = None
api_keys_file = self.api_keys_file
else:
data_folder = os.path.dirname(self.api_keys_file)
api_keys_file = os.path.basename(self.api_keys_file)
self.db.add_filler(
github_rest.ForksFiller(fail_on_wait=self.fail_on_wait,
workers=self.workers,
data_folder=data_folder,
api_keys_file=api_keys_file))
self.db.add_filler(
generic.ClonesFiller()
) # Clones after forks to have up-to-date repo URLS (detect redirects)
self.db.add_filler(
commit_info.CommitsFiller()
) # Commits after forks because fork info needed for repo commit ownership
self.db.add_filler(
github_rest.GHLoginsFiller(fail_on_wait=self.fail_on_wait,
workers=self.workers,
data_folder=data_folder,
api_keys_file=api_keys_file))
self.db.add_filler(
github_rest.StarsFiller(fail_on_wait=self.fail_on_wait,
workers=self.workers,
data_folder=data_folder,
api_keys_file=api_keys_file))
self.db.add_filler(
github_rest.FollowersFiller(fail_on_wait=self.fail_on_wait,
workers=self.workers,
data_folder=data_folder,
api_keys_file=api_keys_file))
def test_gitlab(testdb): testdb.add_filler(generic.SourcesFiller(source=['Gitlab',],source_urlroot=['gitlab.com',])) testdb.add_filler(generic.PackageFiller(package_list_file='packages_gitlab.csv',data_folder=os.path.join(os.path.dirname(__file__),'dummy_data'))) testdb.add_filler(generic.RepositoriesFiller()) testdb.add_filler(generic.ClonesFiller(data_folder='dummy_clones')) # Clones after forks to have up-to-date repo URLS (detect redirects) testdb.add_filler(commit_info.CommitsFiller(data_folder='dummy_clones')) # Commits after forks because fork info needed for repo commit ownership testdb.add_filler(gitlab_gql.LoginsFiller(fail_on_wait=True,workers=workers)) testdb.fill_db()
def test_repositories(testdb):
testdb.add_filler(
generic.SourcesFiller(source=[
'GitHub',
],
source_urlroot=[
'github.com',
]))
testdb.add_filler(generic.PackageFiller(package_list_file='packages.csv'))
testdb.add_filler(generic.RepositoriesFiller())
testdb.fill_db()
def test_commits(testdb):
testdb.add_filler(
generic.SourcesFiller(source=[
'GitHub',
],
source_urlroot=[
'github.com',
]))
testdb.add_filler(generic.PackageFiller(package_list_file='packages.csv'))
testdb.add_filler(generic.RepositoriesFiller())
testdb.add_filler(generic.ClonesFiller(data_folder='dummy_clones'))
testdb.add_filler(commit_info.CommitsFiller(data_folder='dummy_clones'))
testdb.fill_db()
def test_clones_https(testdb):
testdb.add_filler(
generic.SourcesFiller(source=[
'GitHub',
],
source_urlroot=[
'github.com',
]))
testdb.add_filler(generic.PackageFiller(package_list_file='packages.csv'))
testdb.add_filler(generic.RepositoriesFiller())
testdb.add_filler(generic.ClonesFiller(data_folder='dummy_clones'))
# testdb.add_filler(generic.ClonesFiller(data_folder='dummy_clones',rm_first=True))
testdb.fill_db()
def test_repositories(testdb):
testdb.add_filler(
generic.SourcesFiller(source=[
'GitHub',
],
source_urlroot=[
'github.com',
]))
testdb.add_filler(
generic.PackageFiller(package_list_file='packages.csv',
data_folder=os.path.join(
os.path.dirname(__file__), 'dummy_data')))
testdb.add_filler(generic.RepositoriesFiller())
testdb.fill_db()
def test_github(testdb):
testdb.add_filler(
generic.SourcesFiller(source=[
'GitHub',
],
source_urlroot=[
'github.com',
]))
testdb.add_filler(generic.PackageFiller(package_list_file='packages.csv'))
testdb.add_filler(generic.RepositoriesFiller())
testdb.add_filler(generic.ClonesFiller(data_folder='dummy_clones'))
testdb.add_filler(github.ForksFiller(fail_on_wait=True, workers=2))
testdb.add_filler(commit_info.CommitsFiller(data_folder='dummy_clones'))
testdb.add_filler(github.GHLoginsFiller(fail_on_wait=True, workers=2))
testdb.add_filler(github.StarsFiller(fail_on_wait=True, workers=2))
testdb.add_filler(github.FollowersFiller(fail_on_wait=True, workers=2))
testdb.fill_db()
def test_clones_https(testdb):
testdb.add_filler(
generic.SourcesFiller(source=[
'GitHub',
],
source_urlroot=[
'github.com',
]))
testdb.add_filler(
generic.PackageFiller(package_list_file='packages.csv',
data_folder=os.path.join(
os.path.dirname(__file__), 'dummy_data')))
testdb.add_filler(generic.RepositoriesFiller())
testdb.add_filler(
generic.ClonesFiller(data_folder='dummy_clones', update=True))
# testdb.add_filler(generic.ClonesFiller(data_folder='dummy_clones',rm_first=True))
testdb.fill_db()
def test_github_gql(testdb): testdb.add_filler(generic.SourcesFiller(source=['GitHub',],source_urlroot=['github.com',])) testdb.add_filler(generic.PackageFiller(package_list_file='packages.csv',data_folder=os.path.join(os.path.dirname(__file__),'dummy_data'))) testdb.add_filler(generic.RepositoriesFiller()) testdb.add_filler(github_rest.ForksFiller(fail_on_wait=True,workers=workers,no_unauth=True)) testdb.add_filler(generic.ClonesFiller(data_folder='dummy_clones')) # Clones after forks to have up-to-date repo URLS (detect redirects) testdb.add_filler(commit_info.CommitsFiller(data_folder='dummy_clones',force=True,allbranches=True)) # Commits after forks because fork info needed for repo commit ownership testdb.add_filler(github_gql.LoginsGQLFiller(fail_on_wait=True,workers=workers)) testdb.add_filler(github_gql.StarsGQLFiller(fail_on_wait=True,workers=workers)) testdb.add_filler(github_gql.ReleasesGQLFiller(fail_on_wait=True,workers=workers)) testdb.add_filler(github_gql.FollowersGQLFiller(fail_on_wait=True,workers=workers)) testdb.add_filler(github_gql.LanguagesGQLFiller(fail_on_wait=True,workers=workers)) testdb.add_filler(github_gql.SponsorsUserFiller(fail_on_wait=True,workers=workers)) testdb.add_filler(github_gql.IssuesGQLFiller(fail_on_wait=True,workers=workers)) testdb.add_filler(github_gql.BackwardsSponsorsUserFiller(fail_on_wait=True,workers=workers)) testdb.add_filler(generic.RepoCommitOwnershipFiller()) # Clones after forks to have up-to-date repo URLS (detect redirects) testdb.add_filler(snowball.UserSnowballFiller()) testdb.fill_db()
def test_reset_merged_identities(testdb):
testdb.add_filler(
generic.SourcesFiller(source=[
'GitHub',
],
source_urlroot=[
'github.com',
]))
testdb.add_filler(
generic.PackageFiller(package_list_file='packages.csv',
data_folder=os.path.join(
os.path.dirname(__file__), 'dummy_data')))
testdb.add_filler(generic.RepositoriesFiller())
testdb.add_filler(
github_rest.ForksFiller(fail_on_wait=True,
workers=workers,
no_unauth=True))
testdb.add_filler(
generic.ClonesFiller(data_folder='dummy_clones')
) # Clones after forks to have up-to-date repo URLS (detect redirects)
testdb.add_filler(
commit_info.CommitsFiller(data_folder='dummy_clones')
) # Commits after forks because fork info needed for repo commit ownership
testdb.add_filler(
github_rest.GHLoginsFiller(fail_on_wait=True,
workers=workers,
no_unauth=True))
testdb.fill_db()
count = testdb.count_users()
testdb.reset_merged_identities()
assert testdb.count_users() == testdb.count_identities(
), 'There should be as many users as identities'
testdb.fillers = []
testdb.add_filler(
github_rest.GHLoginsFiller(fail_on_wait=True,
workers=workers,
force=True,
no_unauth=True))
testdb.fill_db()
assert testdb.count_users() == count
def test_merge_repositories(testdb):
testdb.add_filler(
generic.SourcesFiller(source=[
'GitHub',
],
source_urlroot=[
'github.com',
]))
testdb.add_filler(
generic.PackageFiller(package_list_file='packages.csv',
data_folder=os.path.join(
os.path.dirname(__file__), 'dummy_data')))
testdb.add_filler(generic.RepositoriesFiller())
testdb.add_filler(generic.ClonesFiller(data_folder='dummy_clones'))
testdb.add_filler(commit_info.CommitsFiller(data_folder='dummy_clones'))
testdb.fill_db()
testdb.plan_repo_merge(new_id=None,
new_owner='blah',
new_name='blih',
obsolete_owner='wschuell',
obsolete_name='experiment_manager',
obsolete_source='GitHub')
testdb.batch_merge_repos()
def test_packages(testdb):
testdb.add_filler(generic.PackageFiller(package_list_file='packages.csv'))
testdb.fill_db()
def test_packages(testdb):
testdb.add_filler(
generic.PackageFiller(package_list_file='packages.csv',
data_folder=os.path.join(
os.path.dirname(__file__), 'dummy_data')))
testdb.fill_db()
print(
'Make sure you have a github API key (with permission read:user for GraphQL) in $HOME/.repo_tools/github_api_keys.txt. Continuing in 3s.'
)
time.sleep(3)
workers = 3 # Number of parallel threads for querying the github APIs
testdb.add_filler(
generic.SourcesFiller(source=[
'GitHub',
], source_urlroot=[
'github.com',
]))
testdb.add_filler(
generic.PackageFiller(package_list_file='packages.csv',
data_folder=os.path.join(
os.path.dirname(os.path.dirname(rp.__file__)),
'tests', 'testmodule', 'dummy_data')))
testdb.add_filler(
generic.RepositoriesFiller()
) # Parses the URLs of the packages to attribute them to the available sources (here only github.com)
testdb.add_filler(github_rest.ForksFiller(fail_on_wait=True, workers=workers))
testdb.add_filler(
generic.ClonesFiller(data_folder='dummy_clones')
) # Clones after forks to have up-to-date repo URLS (detect redirects)
testdb.add_filler(
commit_info.CommitsFiller(data_folder='dummy_clones')
) # Commits after forks because fork info needed for repo commit ownership ran at the end.
testdb.add_filler(
generic.RepoCommitOwnershipFiller()
) # associating repositories as owners of commits (for those who could not be disambiguated using forks) based on creation date of associated package
testdb.add_filler(