def issue_comments(db_path, repo, issue, auth): "Retrieve issue comments for a specific repository" db = sqlite_utils.Database(db_path) token = load_token(auth) for comment in utils.fetch_issue_comments(repo, token, issue): utils.save_issue_comment(db, comment) utils.ensure_db_shape(db)
def stargazers(db_path, repos, auth): "Fetch the users that have starred the specified repositories" db = sqlite_utils.Database(db_path) token = load_token(auth) for repo in repos: full_repo = utils.fetch_repo(repo, token=token) repo_id = utils.save_repo(db, full_repo) stargazers = utils.fetch_stargazers(repo, token) utils.save_stargazers(db, repo_id, stargazers) utils.ensure_db_shape(db)
def workflows(db_path, repos, auth): "Fetch details of GitHub Actions workflows for the specified repositories" db = sqlite_utils.Database(db_path) token = load_token(auth) for repo in repos: full_repo = utils.fetch_repo(repo, token=token) repo_id = utils.save_repo(db, full_repo) workflows = utils.fetch_workflows(token, full_repo["full_name"]) for filename, content in workflows.items(): utils.save_workflow(db, repo_id, filename, content) utils.ensure_db_shape(db)
def contributors(db_path, repos, auth): "Save contributors for the specified repos" db = sqlite_utils.Database(db_path) token = load_token(auth) for repo in repos: repo_full = utils.fetch_repo(repo, token) utils.save_repo(db, repo_full) contributors = utils.fetch_contributors(repo, token) utils.save_contributors(db, contributors, repo_full["id"]) time.sleep(1) utils.ensure_db_shape(db)
def issues(db_path, repo, issue, auth, load): "Save issues for a specified repository, e.g. simonw/datasette" db = sqlite_utils.Database(db_path) token = load_token(auth) repo_full = utils.fetch_repo(repo, token) if load: issues = json.load(open(load)) else: issues = utils.fetch_issues(repo, token, issue) issues = list(issues) utils.save_issues(db, issues, repo_full) utils.ensure_db_shape(db)
def tags(db_path, repos, auth): "Save tags for the specified repos" db = sqlite_utils.Database(db_path) token = load_token(auth) first = True for repo in repos: if not first: time.sleep(1) first = False repo_full = utils.fetch_repo(repo, token) utils.save_repo(db, repo_full) tags = utils.fetch_tags(repo, token) utils.save_tags(db, tags, repo_full["id"]) utils.ensure_db_shape(db)
def pull_requests(db_path, repo, pull_request_ids, auth, load): "Save pull_requests for a specified repository, e.g. simonw/datasette" db = sqlite_utils.Database(db_path) token = load_token(auth) repo_full = utils.fetch_repo(repo, token) utils.save_repo(db, repo_full) if load: pull_requests = json.load(open(load)) else: pull_requests = utils.fetch_pull_requests(repo, token, pull_request_ids) pull_requests = list(pull_requests) utils.save_pull_requests(db, pull_requests, repo_full) utils.ensure_db_shape(db)
def starred(db_path, username, auth, load): "Save repos starred by the specified (or authenticated) username" db = sqlite_utils.Database(db_path) token = load_token(auth) if load: stars = json.load(open(load)) else: stars = utils.fetch_all_starred(username, token) # Which user are we talking about here? if username: user = utils.fetch_user(username, token) else: user = utils.fetch_user(token=token) utils.save_stars(db, user, stars) utils.ensure_db_shape(db)
def scrape_dependents(db_path, repos, auth, verbose): "Scrape dependents for specified repos" try: import bs4 except ImportError: raise click.ClickException( "Optional dependency bs4 is needed for this command") db = sqlite_utils.Database(db_path) token = load_token(auth) for repo in repos: repo_full = utils.fetch_repo(repo, token) utils.save_repo(db, repo_full) for dependent_repo in utils.scrape_dependents(repo, verbose): # Don't fetch repo details if it's already in our DB existing = list(db["repos"].rows_where("full_name = ?", [dependent_repo])) dependent_id = None if not existing: dependent_full = utils.fetch_repo(dependent_repo, token) time.sleep(1) utils.save_repo(db, dependent_full) dependent_id = dependent_full["id"] else: dependent_id = existing[0]["id"] # Only insert if it isn't already there: if not db["dependents"].exists() or not list( db["dependents"].rows_where( "repo = ? and dependent = ?", [repo_full["id"], dependent_id])): db["dependents"].insert( { "repo": repo_full["id"], "dependent": dependent_id, "first_seen_utc": datetime.datetime.utcnow().isoformat(), }, pk=("repo", "dependent"), foreign_keys=( ("repo", "repos", "id"), ("dependent", "repos", "id"), ), ) utils.ensure_db_shape(db)
def repos(db_path, usernames, auth, repo, load): "Save repos owened by the specified (or authenticated) username or organization" db = sqlite_utils.Database(db_path) token = load_token(auth) if load: for loaded_repo in json.load(open(load)): utils.save_repo(db, loaded_repo) else: if repo: # Just these repos for full_name in repo: utils.save_repo(db, utils.fetch_repo(full_name, token)) else: if not usernames: usernames = [None] for username in usernames: for repo in utils.fetch_all_repos(username, token): utils.save_repo(db, repo) utils.ensure_db_shape(db)
def db(): db = sqlite_utils.Database(memory=True) db["repos"].insert( { "id": 1, "full_name": "dogsheep/github-to-sqlite" }, pk="id", columns={ "organization": int, "topics": str, "name": str, "description": str }, ) db["issues"].insert( { "id": 103, "number": 3, "repo": 1 }, pk="id", columns={ "user": int, "assignee": int, "milestone": int, "repo": int, "title": str, "body": str, }, ) issue_comments = json.load( open(pathlib.Path(__file__).parent / "issue-comments.json")) for comment in issue_comments: utils.save_issue_comment(db, comment) utils.ensure_db_shape(db) return db
def commits(db_path, repos, all, auth): "Save commits for the specified repos" db = sqlite_utils.Database(db_path) token = load_token(auth) def stop_when(commit): try: db["commits"].get(commit["sha"]) return True except sqlite_utils.db.NotFoundError: return False if all: stop_when = None for repo in repos: repo_full = utils.fetch_repo(repo, token) utils.save_repo(db, repo_full) commits = utils.fetch_commits(repo, token, stop_when) utils.save_commits(db, commits, repo_full["id"]) time.sleep(1) utils.ensure_db_shape(db)
def db(workflow_yaml, repo): db = sqlite_utils.Database(memory=True) utils.save_repo(db, repo) utils.save_workflow(db, repo["id"], "deploy_demo.yml", workflow_yaml) utils.ensure_db_shape(db) return db
def db(releases, repo): db = sqlite_utils.Database(memory=True) utils.save_repo(db, repo) utils.save_releases(db, releases, repo["id"]) utils.ensure_db_shape(db) return db
def db(stargazers, repo): db = sqlite_utils.Database(memory=True) utils.save_repo(db, repo) utils.save_stargazers(db, repo["id"], stargazers) utils.ensure_db_shape(db) return db
def db(starred, user): db = sqlite_utils.Database(memory=True) utils.save_stars(db, user, starred) utils.ensure_db_shape(db) return db