def contributors(db_path, repos, auth): "Save contributors for the specified repos" db = sqlite_utils.Database(db_path) token = load_token(auth) for repo in repos: repo_full = utils.fetch_repo(repo, token) utils.save_repo(db, repo_full) contributors = utils.fetch_contributors(repo, token) utils.save_contributors(db, contributors, repo_full["id"]) time.sleep(1) utils.ensure_db_shape(db)
def releases(db_path, repos, auth): "Save releases for the specified repos" db = sqlite_utils.Database(db_path) token = load_token(auth) for repo in repos: repo_full = utils.fetch_repo(repo, token) utils.save_repo(db, repo_full) releases = utils.fetch_releases(repo, token) utils.save_releases(db, releases, repo_full["id"]) time.sleep(1) utils.ensure_releases_fts(db)
def issues(db_path, repo, issue_ids, auth, load): "Save issues for a specified repository, e.g. simonw/datasette" db = sqlite_utils.Database(db_path) token = load_token(auth) repo_full = utils.fetch_repo(repo, token) utils.save_repo(db, repo_full) if load: issues = json.load(open(load)) else: issues = utils.fetch_issues(repo, token, issue_ids) issues = list(issues) utils.save_issues(db, issues, repo_full) utils.ensure_db_shape(db)
def tags(db_path, repos, auth): "Save tags for the specified repos" db = sqlite_utils.Database(db_path) token = load_token(auth) first = True for repo in repos: if not first: time.sleep(1) first = False repo_full = utils.fetch_repo(repo, token) utils.save_repo(db, repo_full) tags = utils.fetch_tags(repo, token) utils.save_tags(db, tags, repo_full["id"]) utils.ensure_db_shape(db)
def pull_requests(db_path, repo, pull_request_ids, auth, load): "Save pull_requests for a specified repository, e.g. simonw/datasette" db = sqlite_utils.Database(db_path) token = load_token(auth) repo_full = utils.fetch_repo(repo, token) utils.save_repo(db, repo_full) if load: pull_requests = json.load(open(load)) else: pull_requests = utils.fetch_pull_requests(repo, token, pull_request_ids) pull_requests = list(pull_requests) utils.save_pull_requests(db, pull_requests, repo_full) utils.ensure_db_shape(db)
def repos(db_path, usernames, auth, load): "Save repos owened by the specified (or authenticated) username or organization" db = sqlite_utils.Database(db_path) token = load_token(auth) if load: for repo in json.load(open(load)): utils.save_repo(db, repo) else: if not usernames: usernames = [None] for username in usernames: for repo in utils.fetch_all_repos(username, token): utils.save_repo(db, repo) utils.ensure_repo_fts(db) utils.ensure_foreign_keys(db)
def scrape_dependents(db_path, repos, auth, verbose): "Scrape dependents for specified repos" try: import bs4 except ImportError: raise click.ClickException( "Optional dependency bs4 is needed for this command") db = sqlite_utils.Database(db_path) token = load_token(auth) for repo in repos: repo_full = utils.fetch_repo(repo, token) utils.save_repo(db, repo_full) for dependent_repo in utils.scrape_dependents(repo, verbose): # Don't fetch repo details if it's already in our DB existing = list(db["repos"].rows_where("full_name = ?", [dependent_repo])) dependent_id = None if not existing: dependent_full = utils.fetch_repo(dependent_repo, token) time.sleep(1) utils.save_repo(db, dependent_full) dependent_id = dependent_full["id"] else: dependent_id = existing[0]["id"] # Only insert if it isn't already there: if not db["dependents"].exists() or not list( db["dependents"].rows_where( "repo = ? and dependent = ?", [repo_full["id"], dependent_id])): db["dependents"].insert( { "repo": repo_full["id"], "dependent": dependent_id, "first_seen_utc": datetime.datetime.utcnow().isoformat(), }, pk=("repo", "dependent"), foreign_keys=( ("repo", "repos", "id"), ("dependent", "repos", "id"), ), ) utils.ensure_db_shape(db)
def stargazers(db_path, repos, auth): "Fetch the users that have starred the specified repositories" db = sqlite_utils.Database(db_path) token = load_token(auth) for repo in repos: full_repo = utils.fetch_repo(repo, token=token) repo_id = utils.save_repo(db, full_repo) stargazers = utils.fetch_stargazers(repo, token) utils.save_stargazers(db, repo_id, stargazers) utils.ensure_db_shape(db)
def workflows(db_path, repos, auth): "Fetch details of GitHub Actions workflows for the specified repositories" db = sqlite_utils.Database(db_path) token = load_token(auth) for repo in repos: full_repo = utils.fetch_repo(repo, token=token) repo_id = utils.save_repo(db, full_repo) workflows = utils.fetch_workflows(token, full_repo["full_name"]) for filename, content in workflows.items(): utils.save_workflow(db, repo_id, filename, content) utils.ensure_db_shape(db)
def commits(db_path, repos, all, auth): "Save commits for the specified repos" db = sqlite_utils.Database(db_path) token = load_token(auth) def stop_when(commit): try: db["commits"].get(commit["sha"]) return True except sqlite_utils.db.NotFoundError: return False if all: stop_when = None for repo in repos: repo_full = utils.fetch_repo(repo, token) utils.save_repo(db, repo_full) commits = utils.fetch_commits(repo, token, stop_when) utils.save_commits(db, commits, repo_full["id"]) time.sleep(1)
def repos(db_path, usernames, auth, repo, load, readme, readme_html): "Save repos owened by the specified (or authenticated) username or organization" db = sqlite_utils.Database(db_path) token = load_token(auth) if load: for loaded_repo in json.load(open(load)): utils.save_repo(db, loaded_repo) else: if repo: # Just these repos for full_name in repo: repo_id = utils.save_repo(db, utils.fetch_repo(full_name, token)) _repo_readme(db, token, repo_id, full_name, readme, readme_html) else: if not usernames: usernames = [None] for username in usernames: for repo in utils.fetch_all_repos(username, token): repo_id = utils.save_repo(db, repo) _repo_readme( db, token, repo_id, repo["full_name"], readme, readme_html ) utils.ensure_db_shape(db)
def db(workflow_yaml, repo): db = sqlite_utils.Database(memory=True) utils.save_repo(db, repo) utils.save_workflow(db, repo["id"], "deploy_demo.yml", workflow_yaml) utils.ensure_db_shape(db) return db
def db(commits, repo): db = sqlite_utils.Database(memory=True) utils.save_repo(db, repo) utils.save_commits(db, commits, repo["id"]) return db
def db(tags, repo): db = sqlite_utils.Database(memory=True) utils.save_repo(db, repo) utils.save_tags(db, tags, repo["id"]) return db
def db(releases, repo): db = sqlite_utils.Database(memory=True) utils.save_repo(db, repo) utils.save_releases(db, releases, repo["id"]) utils.ensure_db_shape(db) return db
def db(stargazers, repo): db = sqlite_utils.Database(memory=True) utils.save_repo(db, repo) utils.save_stargazers(db, repo["id"], stargazers) utils.ensure_db_shape(db) return db