def get_issue_events(show_progress):
# Retrieve the list of all projects for which issue events should be fetched.
github_projects = (
GitHubProject
.select(GitHubProject.owner, GitHubProject.repo)
.group_by(GitHubProject.owner, GitHubProject.repo)
)
if show_progress:
progress_bar = ProgressBar(maxval=github_projects.count(), widgets=[
'Progress: ', Percentage(),
' ', Bar(marker=RotatingMarker()),
' ', ETA(),
' Processing project ', Counter(), ' / ' + str(github_projects.count()) + '.'
])
progress_bar.start()
# Create a new fetch index
last_fetch_index = IssueEvent.select(fn.Max(IssueEvent.fetch_index)).scalar() or 0
fetch_index = last_fetch_index + 1
issue_fetch_index = Issue.select(fn.Max(Issue.fetch_index)).scalar() or 0
for project_index, project in enumerate(github_projects, start=1):
# Wrap a callback that will save fetched events
save_events_callback = functools.partial(
save_events,
project=project,
fetch_index=fetch_index,
issue_fetch_index=issue_fetch_index,
)
# Fetch all events for all issues for the project
github_get(
start_url=(
GITHUB_API_URL + '/repos/' + project.owner + '/' +
project.repo + '/issues/events'
),
results_callback=save_events_callback,
)
if show_progress:
progress_bar.update(project_index)
if show_progress:
progress_bar.finish()
# Make sure that all records from the batch inserter have been written out
batch_inserter.flush()
