def get_issue_events(show_progress):
# Retrieve the list of all projects for which issue events should be fetched.
github_projects = (
GitHubProject
.select(GitHubProject.owner, GitHubProject.repo)
.group_by(GitHubProject.owner, GitHubProject.repo)
)
if show_progress:
progress_bar = ProgressBar(maxval=github_projects.count(), widgets=[
'Progress: ', Percentage(),
' ', Bar(marker=RotatingMarker()),
' ', ETA(),
' Processing project ', Counter(), ' / ' + str(github_projects.count()) + '.'
])
progress_bar.start()
# Create a new fetch index
last_fetch_index = IssueEvent.select(fn.Max(IssueEvent.fetch_index)).scalar() or 0
fetch_index = last_fetch_index + 1
issue_fetch_index = Issue.select(fn.Max(Issue.fetch_index)).scalar() or 0
for project_index, project in enumerate(github_projects, start=1):
# Wrap a callback that will save fetched events
save_events_callback = functools.partial(
save_events,
project=project,
fetch_index=fetch_index,
issue_fetch_index=issue_fetch_index,
)
# Fetch all events for all issues for the project
github_get(
start_url=(
GITHUB_API_URL + '/repos/' + project.owner + '/' +
project.repo + '/issues/events'
),
results_callback=save_events_callback,
)
if show_progress:
progress_bar.update(project_index)
if show_progress:
progress_bar.finish()
# Make sure that all records from the batch inserter have been written out
batch_inserter.flush()
def get_issues_for_projects(projects, show_progress):
if show_progress:
progress_bar = ProgressBar(maxval=len(projects), widgets=[
'Progress: ', Percentage(),
' ', Bar(marker=RotatingMarker()),
' ', ETA(),
' Processing project ', Counter(), ' / ' + str(len(projects)) + '.'
])
progress_bar.start()
# Create a new fetch index
last_fetch_index = GitHubProject.select(fn.Max(GitHubProject.fetch_index)).scalar() or 0
fetch_index = last_fetch_index + 1
for project_index, project in enumerate(projects, start=1):
# Create a new record for the project based on the JSON specification
project = GitHubProject.create(
fetch_index=fetch_index,
name=project['name'],
owner=project['owner'],
repo=project['repo'],
)
# Wrap a callback that will save issues associated to this project and fetch index.
save_issues_callback = functools.partial(
save_issues,
project=project,
fetch_index=fetch_index,
)
# Fetch all issues for the project from GitHub
github_get(
start_url=GITHUB_API_URL + '/repos/' + project.owner + '/' + project.repo + '/issues',
results_callback=save_issues_callback,
params={
'state': 'all',
}
)
if show_progress:
progress_bar.update(project_index)
if show_progress:
progress_bar.finish()