def contributors(db_path, repos, auth):
"Save contributors for the specified repos"
db = sqlite_utils.Database(db_path)
token = load_token(auth)
for repo in repos:
repo_full = utils.fetch_repo(repo, token)
utils.save_repo(db, repo_full)
contributors = utils.fetch_contributors(repo, token)
utils.save_contributors(db, contributors, repo_full["id"])
time.sleep(1)
utils.ensure_db_shape(db)
def releases(db_path, repos, auth):
"Save releases for the specified repos"
db = sqlite_utils.Database(db_path)
token = load_token(auth)
for repo in repos:
repo_full = utils.fetch_repo(repo, token)
utils.save_repo(db, repo_full)
releases = utils.fetch_releases(repo, token)
utils.save_releases(db, releases, repo_full["id"])
time.sleep(1)
utils.ensure_releases_fts(db)
def issues(db_path, repo, issue_ids, auth, load):
"Save issues for a specified repository, e.g. simonw/datasette"
db = sqlite_utils.Database(db_path)
token = load_token(auth)
repo_full = utils.fetch_repo(repo, token)
utils.save_repo(db, repo_full)
if load:
issues = json.load(open(load))
else:
issues = utils.fetch_issues(repo, token, issue_ids)
issues = list(issues)
utils.save_issues(db, issues, repo_full)
utils.ensure_db_shape(db)
def tags(db_path, repos, auth):
"Save tags for the specified repos"
db = sqlite_utils.Database(db_path)
token = load_token(auth)
first = True
for repo in repos:
if not first:
time.sleep(1)
first = False
repo_full = utils.fetch_repo(repo, token)
utils.save_repo(db, repo_full)
tags = utils.fetch_tags(repo, token)
utils.save_tags(db, tags, repo_full["id"])
utils.ensure_db_shape(db)
def pull_requests(db_path, repo, pull_request_ids, auth, load):
"Save pull_requests for a specified repository, e.g. simonw/datasette"
db = sqlite_utils.Database(db_path)
token = load_token(auth)
repo_full = utils.fetch_repo(repo, token)
utils.save_repo(db, repo_full)
if load:
pull_requests = json.load(open(load))
else:
pull_requests = utils.fetch_pull_requests(repo, token, pull_request_ids)
pull_requests = list(pull_requests)
utils.save_pull_requests(db, pull_requests, repo_full)
utils.ensure_db_shape(db)
def repos(db_path, usernames, auth, load):
"Save repos owened by the specified (or authenticated) username or organization"
db = sqlite_utils.Database(db_path)
token = load_token(auth)
if load:
for repo in json.load(open(load)):
utils.save_repo(db, repo)
else:
if not usernames:
usernames = [None]
for username in usernames:
for repo in utils.fetch_all_repos(username, token):
utils.save_repo(db, repo)
utils.ensure_repo_fts(db)
utils.ensure_foreign_keys(db)
def scrape_dependents(db_path, repos, auth, verbose):
"Scrape dependents for specified repos"
try:
import bs4
except ImportError:
raise click.ClickException(
"Optional dependency bs4 is needed for this command")
db = sqlite_utils.Database(db_path)
token = load_token(auth)
for repo in repos:
repo_full = utils.fetch_repo(repo, token)
utils.save_repo(db, repo_full)
for dependent_repo in utils.scrape_dependents(repo, verbose):
# Don't fetch repo details if it's already in our DB
existing = list(db["repos"].rows_where("full_name = ?",
[dependent_repo]))
dependent_id = None
if not existing:
dependent_full = utils.fetch_repo(dependent_repo, token)
time.sleep(1)
utils.save_repo(db, dependent_full)
dependent_id = dependent_full["id"]
else:
dependent_id = existing[0]["id"]
# Only insert if it isn't already there:
if not db["dependents"].exists() or not list(
db["dependents"].rows_where(
"repo = ? and dependent = ?",
[repo_full["id"], dependent_id])):
db["dependents"].insert(
{
"repo": repo_full["id"],
"dependent": dependent_id,
"first_seen_utc":
datetime.datetime.utcnow().isoformat(),
},
pk=("repo", "dependent"),
foreign_keys=(
("repo", "repos", "id"),
("dependent", "repos", "id"),
),
)
utils.ensure_db_shape(db)
def stargazers(db_path, repos, auth):
"Fetch the users that have starred the specified repositories"
db = sqlite_utils.Database(db_path)
token = load_token(auth)
for repo in repos:
full_repo = utils.fetch_repo(repo, token=token)
repo_id = utils.save_repo(db, full_repo)
stargazers = utils.fetch_stargazers(repo, token)
utils.save_stargazers(db, repo_id, stargazers)
utils.ensure_db_shape(db)
def workflows(db_path, repos, auth):
"Fetch details of GitHub Actions workflows for the specified repositories"
db = sqlite_utils.Database(db_path)
token = load_token(auth)
for repo in repos:
full_repo = utils.fetch_repo(repo, token=token)
repo_id = utils.save_repo(db, full_repo)
workflows = utils.fetch_workflows(token, full_repo["full_name"])
for filename, content in workflows.items():
utils.save_workflow(db, repo_id, filename, content)
utils.ensure_db_shape(db)
def commits(db_path, repos, all, auth):
"Save commits for the specified repos"
db = sqlite_utils.Database(db_path)
token = load_token(auth)
def stop_when(commit):
try:
db["commits"].get(commit["sha"])
return True
except sqlite_utils.db.NotFoundError:
return False
if all:
stop_when = None
for repo in repos:
repo_full = utils.fetch_repo(repo, token)
utils.save_repo(db, repo_full)
commits = utils.fetch_commits(repo, token, stop_when)
utils.save_commits(db, commits, repo_full["id"])
time.sleep(1)
def repos(db_path, usernames, auth, repo, load, readme, readme_html):
"Save repos owened by the specified (or authenticated) username or organization"
db = sqlite_utils.Database(db_path)
token = load_token(auth)
if load:
for loaded_repo in json.load(open(load)):
utils.save_repo(db, loaded_repo)
else:
if repo:
# Just these repos
for full_name in repo:
repo_id = utils.save_repo(db, utils.fetch_repo(full_name, token))
_repo_readme(db, token, repo_id, full_name, readme, readme_html)
else:
if not usernames:
usernames = [None]
for username in usernames:
for repo in utils.fetch_all_repos(username, token):
repo_id = utils.save_repo(db, repo)
_repo_readme(
db, token, repo_id, repo["full_name"], readme, readme_html
)
utils.ensure_db_shape(db)
def db(workflow_yaml, repo):
db = sqlite_utils.Database(memory=True)
utils.save_repo(db, repo)
utils.save_workflow(db, repo["id"], "deploy_demo.yml", workflow_yaml)
utils.ensure_db_shape(db)
return db
def db(commits, repo):
db = sqlite_utils.Database(memory=True)
utils.save_repo(db, repo)
utils.save_commits(db, commits, repo["id"])
return db
def db(tags, repo):
db = sqlite_utils.Database(memory=True)
utils.save_repo(db, repo)
utils.save_tags(db, tags, repo["id"])
return db
def db(releases, repo):
db = sqlite_utils.Database(memory=True)
utils.save_repo(db, repo)
utils.save_releases(db, releases, repo["id"])
utils.ensure_db_shape(db)
return db
def db(stargazers, repo):
db = sqlite_utils.Database(memory=True)
utils.save_repo(db, repo)
utils.save_stargazers(db, repo["id"], stargazers)
utils.ensure_db_shape(db)
return db
