def test_gitlab(testdb): testdb.add_filler(generic.SourcesFiller(source=['Gitlab',],source_urlroot=['gitlab.com',])) testdb.add_filler(generic.PackageFiller(package_list_file='packages_gitlab.csv',data_folder=os.path.join(os.path.dirname(__file__),'dummy_data'))) testdb.add_filler(generic.RepositoriesFiller()) testdb.add_filler(generic.ClonesFiller(data_folder='dummy_clones')) # Clones after forks to have up-to-date repo URLS (detect redirects) testdb.add_filler(commit_info.CommitsFiller(data_folder='dummy_clones')) # Commits after forks because fork info needed for repo commit ownership testdb.add_filler(gitlab_gql.LoginsFiller(fail_on_wait=True,workers=workers)) testdb.fill_db()
def test_sources2(testdb): testdb.add_filler( generic.SourcesFiller(source=[ 'GitHub', ], source_urlroot=[ 'github.com', ])) testdb.fill_db()
def test_repositories(testdb): testdb.add_filler( generic.SourcesFiller(source=[ 'GitHub', ], source_urlroot=[ 'github.com', ])) testdb.add_filler(generic.PackageFiller(package_list_file='packages.csv')) testdb.add_filler(generic.RepositoriesFiller()) testdb.fill_db()
def prepare(self): self.db.add_filler( generic.SourcesFiller(source=[ 'GitHub', ], source_urlroot=[ 'github.com', ])) if os.path.isabs(self.packages_file): self.db.add_filler( generic.PackageFiller( package_list_file=os.path.basename(self.packages_file), data_folder=os.path.dirname(self.packages_file))) else: self.db.add_filler( generic.PackageFiller(package_list_file=self.packages_file)) self.db.add_filler(generic.RepositoriesFiller()) if os.path.isabs(self.api_keys_file): data_folder = None api_keys_file = self.api_keys_file else: data_folder = os.path.dirname(self.api_keys_file) api_keys_file = os.path.basename(self.api_keys_file) self.db.add_filler( github_rest.ForksFiller(fail_on_wait=self.fail_on_wait, workers=self.workers, data_folder=data_folder, api_keys_file=api_keys_file)) self.db.add_filler( generic.ClonesFiller() ) # Clones after forks to have up-to-date repo URLS (detect redirects) self.db.add_filler( commit_info.CommitsFiller() ) # Commits after forks because fork info needed for repo commit ownership self.db.add_filler( github_rest.GHLoginsFiller(fail_on_wait=self.fail_on_wait, workers=self.workers, data_folder=data_folder, api_keys_file=api_keys_file)) self.db.add_filler( github_rest.StarsFiller(fail_on_wait=self.fail_on_wait, workers=self.workers, data_folder=data_folder, api_keys_file=api_keys_file)) self.db.add_filler( github_rest.FollowersFiller(fail_on_wait=self.fail_on_wait, workers=self.workers, data_folder=data_folder, api_keys_file=api_keys_file))
def test_urls(testdb): testdb.add_filler( generic.SourcesFiller(source='GitHub', source_urlroot='github.com')) testdb.add_filler( generic.URLFiller(url_list_file='urls.csv', data_folder=os.path.join(os.path.dirname(__file__), 'dummy_data'))) testdb.add_filler( generic.URLFiller(url_list=['github.com/deepcharles/ruptures'], data_folder=os.path.join(os.path.dirname(__file__), 'dummy_data'))) testdb.fill_db()
def test_urls(testdb): testdb.add_filler( generic.SourcesFiller(source='GitHub', source_urlroot='github.com')) testdb.add_filler( generic.URLFiller(url_list=['github.com/Wschuell/FabSub'], data_folder=os.path.join(os.path.dirname(__file__), 'dummy_data'))) testdb.add_filler(generic.RepositoriesFiller()) testdb.add_filler(generic.ClonesFiller(data_folder='dummy_clones')) testdb.add_filler(commit_info.CommitsFiller(data_folder='dummy_clones')) testdb.add_filler( github_gql.StarsGQLFiller(fail_on_wait=True, workers=workers)) testdb.fill_db()
def test_commits(testdb): testdb.add_filler( generic.SourcesFiller(source=[ 'GitHub', ], source_urlroot=[ 'github.com', ])) testdb.add_filler(generic.PackageFiller(package_list_file='packages.csv')) testdb.add_filler(generic.RepositoriesFiller()) testdb.add_filler(generic.ClonesFiller(data_folder='dummy_clones')) testdb.add_filler(commit_info.CommitsFiller(data_folder='dummy_clones')) testdb.fill_db()
def test_clones_https(testdb): testdb.add_filler( generic.SourcesFiller(source=[ 'GitHub', ], source_urlroot=[ 'github.com', ])) testdb.add_filler(generic.PackageFiller(package_list_file='packages.csv')) testdb.add_filler(generic.RepositoriesFiller()) testdb.add_filler(generic.ClonesFiller(data_folder='dummy_clones')) # testdb.add_filler(generic.ClonesFiller(data_folder='dummy_clones',rm_first=True)) testdb.fill_db()
def test_repositories(testdb): testdb.add_filler( generic.SourcesFiller(source=[ 'GitHub', ], source_urlroot=[ 'github.com', ])) testdb.add_filler( generic.PackageFiller(package_list_file='packages.csv', data_folder=os.path.join( os.path.dirname(__file__), 'dummy_data'))) testdb.add_filler(generic.RepositoriesFiller()) testdb.fill_db()
def test_github(testdb): testdb.add_filler( generic.SourcesFiller(source=[ 'GitHub', ], source_urlroot=[ 'github.com', ])) testdb.add_filler(generic.PackageFiller(package_list_file='packages.csv')) testdb.add_filler(generic.RepositoriesFiller()) testdb.add_filler(generic.ClonesFiller(data_folder='dummy_clones')) testdb.add_filler(github.ForksFiller(fail_on_wait=True, workers=2)) testdb.add_filler(commit_info.CommitsFiller(data_folder='dummy_clones')) testdb.add_filler(github.GHLoginsFiller(fail_on_wait=True, workers=2)) testdb.add_filler(github.StarsFiller(fail_on_wait=True, workers=2)) testdb.add_filler(github.FollowersFiller(fail_on_wait=True, workers=2)) testdb.fill_db()
def test_clones_https(testdb): testdb.add_filler( generic.SourcesFiller(source=[ 'GitHub', ], source_urlroot=[ 'github.com', ])) testdb.add_filler( generic.PackageFiller(package_list_file='packages.csv', data_folder=os.path.join( os.path.dirname(__file__), 'dummy_data'))) testdb.add_filler(generic.RepositoriesFiller()) testdb.add_filler( generic.ClonesFiller(data_folder='dummy_clones', update=True)) # testdb.add_filler(generic.ClonesFiller(data_folder='dummy_clones',rm_first=True)) testdb.fill_db()
def test_github_gql(testdb): testdb.add_filler(generic.SourcesFiller(source=['GitHub',],source_urlroot=['github.com',])) testdb.add_filler(generic.PackageFiller(package_list_file='packages.csv',data_folder=os.path.join(os.path.dirname(__file__),'dummy_data'))) testdb.add_filler(generic.RepositoriesFiller()) testdb.add_filler(github_rest.ForksFiller(fail_on_wait=True,workers=workers,no_unauth=True)) testdb.add_filler(generic.ClonesFiller(data_folder='dummy_clones')) # Clones after forks to have up-to-date repo URLS (detect redirects) testdb.add_filler(commit_info.CommitsFiller(data_folder='dummy_clones',force=True,allbranches=True)) # Commits after forks because fork info needed for repo commit ownership testdb.add_filler(github_gql.LoginsGQLFiller(fail_on_wait=True,workers=workers)) testdb.add_filler(github_gql.StarsGQLFiller(fail_on_wait=True,workers=workers)) testdb.add_filler(github_gql.ReleasesGQLFiller(fail_on_wait=True,workers=workers)) testdb.add_filler(github_gql.FollowersGQLFiller(fail_on_wait=True,workers=workers)) testdb.add_filler(github_gql.LanguagesGQLFiller(fail_on_wait=True,workers=workers)) testdb.add_filler(github_gql.SponsorsUserFiller(fail_on_wait=True,workers=workers)) testdb.add_filler(github_gql.IssuesGQLFiller(fail_on_wait=True,workers=workers)) testdb.add_filler(github_gql.BackwardsSponsorsUserFiller(fail_on_wait=True,workers=workers)) testdb.add_filler(generic.RepoCommitOwnershipFiller()) # Clones after forks to have up-to-date repo URLS (detect redirects) testdb.add_filler(snowball.UserSnowballFiller()) testdb.fill_db()
def test_reset_merged_identities(testdb): testdb.add_filler( generic.SourcesFiller(source=[ 'GitHub', ], source_urlroot=[ 'github.com', ])) testdb.add_filler( generic.PackageFiller(package_list_file='packages.csv', data_folder=os.path.join( os.path.dirname(__file__), 'dummy_data'))) testdb.add_filler(generic.RepositoriesFiller()) testdb.add_filler( github_rest.ForksFiller(fail_on_wait=True, workers=workers, no_unauth=True)) testdb.add_filler( generic.ClonesFiller(data_folder='dummy_clones') ) # Clones after forks to have up-to-date repo URLS (detect redirects) testdb.add_filler( commit_info.CommitsFiller(data_folder='dummy_clones') ) # Commits after forks because fork info needed for repo commit ownership testdb.add_filler( github_rest.GHLoginsFiller(fail_on_wait=True, workers=workers, no_unauth=True)) testdb.fill_db() count = testdb.count_users() testdb.reset_merged_identities() assert testdb.count_users() == testdb.count_identities( ), 'There should be as many users as identities' testdb.fillers = [] testdb.add_filler( github_rest.GHLoginsFiller(fail_on_wait=True, workers=workers, force=True, no_unauth=True)) testdb.fill_db() assert testdb.count_users() == count
def test_merge_repositories(testdb): testdb.add_filler( generic.SourcesFiller(source=[ 'GitHub', ], source_urlroot=[ 'github.com', ])) testdb.add_filler( generic.PackageFiller(package_list_file='packages.csv', data_folder=os.path.join( os.path.dirname(__file__), 'dummy_data'))) testdb.add_filler(generic.RepositoriesFiller()) testdb.add_filler(generic.ClonesFiller(data_folder='dummy_clones')) testdb.add_filler(commit_info.CommitsFiller(data_folder='dummy_clones')) testdb.fill_db() testdb.plan_repo_merge(new_id=None, new_owner='blah', new_name='blih', obsolete_owner='wschuell', obsolete_name='experiment_manager', obsolete_source='GitHub') testdb.batch_merge_repos()
def test_sources(testdb): testdb.add_filler( generic.SourcesFiller(source='GitHub', source_urlroot='github.com')) testdb.fill_db()
testdb = rp.repo_database.Database(db_name='testdb_repotools', db_type='sqlite', data_folder='dummy_clones') testdb.init_db() print( 'Make sure you have a github API key (with permission read:user for GraphQL) in $HOME/.repo_tools/github_api_keys.txt. Continuing in 3s.' ) time.sleep(3) workers = 3 # Number of parallel threads for querying the github APIs testdb.add_filler( generic.SourcesFiller(source=[ 'GitHub', ], source_urlroot=[ 'github.com', ])) testdb.add_filler( generic.PackageFiller(package_list_file='packages.csv', data_folder=os.path.join( os.path.dirname(os.path.dirname(rp.__file__)), 'tests', 'testmodule', 'dummy_data'))) testdb.add_filler( generic.RepositoriesFiller() ) # Parses the URLs of the packages to attribute them to the available sources (here only github.com) testdb.add_filler(github_rest.ForksFiller(fail_on_wait=True, workers=workers)) testdb.add_filler( generic.ClonesFiller(data_folder='dummy_clones') ) # Clones after forks to have up-to-date repo URLS (detect redirects) testdb.add_filler(