def __init__(self,
creds=None,
host='api.github.com',
owner='Pardot',
repo='pardot'):
self._last_date = None
self._last_identifier = None
self._no_more_requests_until = None
github_org = GithubOrg(host, owner, creds)
self._github_repo = GithubRepo(github_org, repo)
def __init__(self,
creds=None,
host='api.github.com',
owner='Salesforce',
repo='providence'):
# default repository to github.com/salesforce/providence
self._last_date = None
self._last_identifier = None
self._no_more_requests_until = None
github_org = GithubOrg(host, owner, creds)
self._github_repo = GithubRepo(github_org, repo)
class GithubSource(RepoSource):
def __init__(self,
creds=None,
host='api.github.com',
owner='Salesforce',
repo='providence'):
# default repository to github.com/salesforce/providence
self._last_date = None
self._last_identifier = None
self._no_more_requests_until = None
github_org = GithubOrg(host, owner, creds)
self._github_repo = GithubRepo(github_org, repo)
def processSinceIdentifier(self,
identifier,
commit_started_callback,
patch_callback,
commit_finished_callback,
path=None):
since_datetime = datetime.datetime.utcnow()
since_datetime = since_datetime.replace(tzinfo=pytz.UTC,
hour=0,
minute=0,
second=0)
if identifier:
try:
since_datetime = dateutil.parser.parse(identifier)
except Exception as e:
logger.error("Error parsing datetime from %s", identifier)
commits = self._github_repo.commits(since_datetime, path=path)
self.processCommits(commits,
commit_started_callback=commit_started_callback,
patch_callback=patch_callback,
commit_finished_callback=commit_finished_callback)
if self._last_date:
return self._last_date.isoformat()
if since_datetime:
return since_datetime.isoformat()
return None
def processCommits(self, commits, commit_started_callback, patch_callback,
commit_finished_callback):
if commits is None:
return None
# Process oldest first
commits = commits[::-1]
logger.debug("processing %d commits", len(commits))
for github_commit in commits:
logger.debug("sha: %s", github_commit.sha)
if github_commit.sha:
github_commit = self._github_repo.commit(github_commit.sha)
repo_commit = RepoCommit()
repo_commit.url = github_commit.html_url
repo_commit.repo_source = self
if github_commit.date is not None:
self._last_date = dateutil.parser.parse(github_commit.date)
repo_commit.date = self._last_date
self._last_date += datetime.timedelta(seconds=1)
repo_commit.identifier = self._last_date.isoformat()
repo_commit.committer_email = github_commit.committer_email
repo_commit.committer_name = github_commit.committer_name
repo_commit.username = github_commit.committer_login
repo_commit.message = github_commit.message
repo_commit.sha = github_commit.sha
commit_started_callback(repo_commit)
if github_commit.files:
for file_info in github_commit.files:
if file_info.get('patch'):
filename = committer_username = None
diff = DiffParser(file_info['patch'])
repo_patch = RepoPatch(repo_commit=repo_commit)
repo_patch.diff = diff
repo_patch.filename = file_info.get("filename")
patch_callback(repo_patch)
commit_finished_callback(repo_commit)
logger.debug("commit sha: %s processing complete",
github_commit.sha)
#logger.debug("done")
def retrieve_file(self, file_path):
file_content = self._github_repo.get_raw_file(file_path)
if file_content is None:
file_content = ''
return file_content
def __init__(self, creds=None, host='api.github.com', owner='Pardot', repo='pardot'):
self._last_date = None
self._last_identifier = None
self._no_more_requests_until = None
github_org = GithubOrg(host, owner, creds)
self._github_repo = GithubRepo(github_org, repo)
class GithubSource(RepoSource):
def __init__(self, creds=None, host='api.github.com', owner='Pardot', repo='pardot'):
self._last_date = None
self._last_identifier = None
self._no_more_requests_until = None
github_org = GithubOrg(host, owner, creds)
self._github_repo = GithubRepo(github_org, repo)
def processSinceIdentifier(self, identifier, commit_started_callback, patch_callback, commit_finished_callback, path=None):
since_datetime = datetime.datetime.utcnow()
since_datetime = since_datetime.replace(tzinfo=pytz.UTC, hour=0, minute=0, second=0)
if identifier:
try:
since_datetime = dateutil.parser.parse(identifier)
except Exception as e:
logger.error("Error parsing datetime from %s", identifier)
commits = self._github_repo.commits(since_datetime, path=path)
self.processCommits(commits,
commit_started_callback=commit_started_callback,
patch_callback=patch_callback,
commit_finished_callback=commit_finished_callback);
if self._last_date:
return self._last_date.isoformat()
if since_datetime:
return since_datetime.isoformat()
return None
def processCommits(self, commits, commit_started_callback, patch_callback, commit_finished_callback):
if commits is None:
return None
# Process oldest first
commits = commits[::-1]
logger.debug("process %d commits", len(commits))
for github_commit in commits:
logger.debug("sha: %s", github_commit.sha)
if github_commit.sha:
github_commit = self._github_repo.commit(github_commit.sha)
repo_commit = RepoCommit();
repo_commit.url = github_commit.html_url
repo_commit.repo_source = self
if github_commit.date is not None:
self._last_date = dateutil.parser.parse(github_commit.date)
repo_commit.date = self._last_date
self._last_date += datetime.timedelta(seconds=1)
repo_commit.identifier = self._last_date.isoformat()
repo_commit.committer_email = github_commit.committer_email
repo_commit.committer_name = github_commit.committer_name
repo_commit.username = github_commit.committer_login
repo_commit.message = github_commit.message
repo_commit.sha = github_commit.sha
commit_started_callback(repo_commit)
if github_commit.files:
for file_info in github_commit.files:
if file_info.get('patch'):
filename = committer_username = None
diff = DiffParser(file_info['patch'])
repo_patch = RepoPatch(repo_commit=repo_commit)
repo_patch.diff = diff
repo_patch.filename = file_info.get("filename")
patch_callback(repo_patch)
commit_finished_callback(repo_commit)
logger.debug("batch fof files processed")
logger.debug("done")
