def issue_comments(db_path, repo, issue, auth):
"Retrieve issue comments for a specific repository"
db = sqlite_utils.Database(db_path)
token = load_token(auth)
for comment in utils.fetch_issue_comments(repo, token, issue):
utils.save_issue_comment(db, comment)
utils.ensure_db_shape(db)
def stargazers(db_path, repos, auth):
"Fetch the users that have starred the specified repositories"
db = sqlite_utils.Database(db_path)
token = load_token(auth)
for repo in repos:
full_repo = utils.fetch_repo(repo, token=token)
repo_id = utils.save_repo(db, full_repo)
stargazers = utils.fetch_stargazers(repo, token)
utils.save_stargazers(db, repo_id, stargazers)
utils.ensure_db_shape(db)
def workflows(db_path, repos, auth):
"Fetch details of GitHub Actions workflows for the specified repositories"
db = sqlite_utils.Database(db_path)
token = load_token(auth)
for repo in repos:
full_repo = utils.fetch_repo(repo, token=token)
repo_id = utils.save_repo(db, full_repo)
workflows = utils.fetch_workflows(token, full_repo["full_name"])
for filename, content in workflows.items():
utils.save_workflow(db, repo_id, filename, content)
utils.ensure_db_shape(db)
def contributors(db_path, repos, auth):
"Save contributors for the specified repos"
db = sqlite_utils.Database(db_path)
token = load_token(auth)
for repo in repos:
repo_full = utils.fetch_repo(repo, token)
utils.save_repo(db, repo_full)
contributors = utils.fetch_contributors(repo, token)
utils.save_contributors(db, contributors, repo_full["id"])
time.sleep(1)
utils.ensure_db_shape(db)
def issues(db_path, repo, issue, auth, load):
"Save issues for a specified repository, e.g. simonw/datasette"
db = sqlite_utils.Database(db_path)
token = load_token(auth)
repo_full = utils.fetch_repo(repo, token)
if load:
issues = json.load(open(load))
else:
issues = utils.fetch_issues(repo, token, issue)
issues = list(issues)
utils.save_issues(db, issues, repo_full)
utils.ensure_db_shape(db)
def tags(db_path, repos, auth):
"Save tags for the specified repos"
db = sqlite_utils.Database(db_path)
token = load_token(auth)
first = True
for repo in repos:
if not first:
time.sleep(1)
first = False
repo_full = utils.fetch_repo(repo, token)
utils.save_repo(db, repo_full)
tags = utils.fetch_tags(repo, token)
utils.save_tags(db, tags, repo_full["id"])
utils.ensure_db_shape(db)
def pull_requests(db_path, repo, pull_request_ids, auth, load):
"Save pull_requests for a specified repository, e.g. simonw/datasette"
db = sqlite_utils.Database(db_path)
token = load_token(auth)
repo_full = utils.fetch_repo(repo, token)
utils.save_repo(db, repo_full)
if load:
pull_requests = json.load(open(load))
else:
pull_requests = utils.fetch_pull_requests(repo, token, pull_request_ids)
pull_requests = list(pull_requests)
utils.save_pull_requests(db, pull_requests, repo_full)
utils.ensure_db_shape(db)
def starred(db_path, username, auth, load):
"Save repos starred by the specified (or authenticated) username"
db = sqlite_utils.Database(db_path)
token = load_token(auth)
if load:
stars = json.load(open(load))
else:
stars = utils.fetch_all_starred(username, token)
# Which user are we talking about here?
if username:
user = utils.fetch_user(username, token)
else:
user = utils.fetch_user(token=token)
utils.save_stars(db, user, stars)
utils.ensure_db_shape(db)
def scrape_dependents(db_path, repos, auth, verbose):
"Scrape dependents for specified repos"
try:
import bs4
except ImportError:
raise click.ClickException(
"Optional dependency bs4 is needed for this command")
db = sqlite_utils.Database(db_path)
token = load_token(auth)
for repo in repos:
repo_full = utils.fetch_repo(repo, token)
utils.save_repo(db, repo_full)
for dependent_repo in utils.scrape_dependents(repo, verbose):
# Don't fetch repo details if it's already in our DB
existing = list(db["repos"].rows_where("full_name = ?",
[dependent_repo]))
dependent_id = None
if not existing:
dependent_full = utils.fetch_repo(dependent_repo, token)
time.sleep(1)
utils.save_repo(db, dependent_full)
dependent_id = dependent_full["id"]
else:
dependent_id = existing[0]["id"]
# Only insert if it isn't already there:
if not db["dependents"].exists() or not list(
db["dependents"].rows_where(
"repo = ? and dependent = ?",
[repo_full["id"], dependent_id])):
db["dependents"].insert(
{
"repo": repo_full["id"],
"dependent": dependent_id,
"first_seen_utc":
datetime.datetime.utcnow().isoformat(),
},
pk=("repo", "dependent"),
foreign_keys=(
("repo", "repos", "id"),
("dependent", "repos", "id"),
),
)
utils.ensure_db_shape(db)
def repos(db_path, usernames, auth, repo, load):
"Save repos owened by the specified (or authenticated) username or organization"
db = sqlite_utils.Database(db_path)
token = load_token(auth)
if load:
for loaded_repo in json.load(open(load)):
utils.save_repo(db, loaded_repo)
else:
if repo:
# Just these repos
for full_name in repo:
utils.save_repo(db, utils.fetch_repo(full_name, token))
else:
if not usernames:
usernames = [None]
for username in usernames:
for repo in utils.fetch_all_repos(username, token):
utils.save_repo(db, repo)
utils.ensure_db_shape(db)
def db():
db = sqlite_utils.Database(memory=True)
db["repos"].insert(
{
"id": 1,
"full_name": "dogsheep/github-to-sqlite"
},
pk="id",
columns={
"organization": int,
"topics": str,
"name": str,
"description": str
},
)
db["issues"].insert(
{
"id": 103,
"number": 3,
"repo": 1
},
pk="id",
columns={
"user": int,
"assignee": int,
"milestone": int,
"repo": int,
"title": str,
"body": str,
},
)
issue_comments = json.load(
open(pathlib.Path(__file__).parent / "issue-comments.json"))
for comment in issue_comments:
utils.save_issue_comment(db, comment)
utils.ensure_db_shape(db)
return db
def commits(db_path, repos, all, auth):
"Save commits for the specified repos"
db = sqlite_utils.Database(db_path)
token = load_token(auth)
def stop_when(commit):
try:
db["commits"].get(commit["sha"])
return True
except sqlite_utils.db.NotFoundError:
return False
if all:
stop_when = None
for repo in repos:
repo_full = utils.fetch_repo(repo, token)
utils.save_repo(db, repo_full)
commits = utils.fetch_commits(repo, token, stop_when)
utils.save_commits(db, commits, repo_full["id"])
time.sleep(1)
utils.ensure_db_shape(db)
def db(workflow_yaml, repo):
db = sqlite_utils.Database(memory=True)
utils.save_repo(db, repo)
utils.save_workflow(db, repo["id"], "deploy_demo.yml", workflow_yaml)
utils.ensure_db_shape(db)
return db
def db(releases, repo):
db = sqlite_utils.Database(memory=True)
utils.save_repo(db, repo)
utils.save_releases(db, releases, repo["id"])
utils.ensure_db_shape(db)
return db
def db(stargazers, repo):
db = sqlite_utils.Database(memory=True)
utils.save_repo(db, repo)
utils.save_stargazers(db, repo["id"], stargazers)
utils.ensure_db_shape(db)
return db
def db(starred, user):
db = sqlite_utils.Database(memory=True)
utils.save_stars(db, user, starred)
utils.ensure_db_shape(db)
return db
