예제 #1
0
 def __init__(self,
              creds=None,
              host='api.github.com',
              owner='Pardot',
              repo='pardot'):
     self._last_date = None
     self._last_identifier = None
     self._no_more_requests_until = None
     github_org = GithubOrg(host, owner, creds)
     self._github_repo = GithubRepo(github_org, repo)
예제 #2
0
 def __init__(self,
              creds=None,
              host='api.github.com',
              owner='Salesforce',
              repo='providence'):
     # default repository to github.com/salesforce/providence
     self._last_date = None
     self._last_identifier = None
     self._no_more_requests_until = None
     github_org = GithubOrg(host, owner, creds)
     self._github_repo = GithubRepo(github_org, repo)
예제 #3
0
class GithubSource(RepoSource):
    def __init__(self,
                 creds=None,
                 host='api.github.com',
                 owner='Salesforce',
                 repo='providence'):
        # default repository to github.com/salesforce/providence
        self._last_date = None
        self._last_identifier = None
        self._no_more_requests_until = None
        github_org = GithubOrg(host, owner, creds)
        self._github_repo = GithubRepo(github_org, repo)

    def processSinceIdentifier(self,
                               identifier,
                               commit_started_callback,
                               patch_callback,
                               commit_finished_callback,
                               path=None):
        since_datetime = datetime.datetime.utcnow()
        since_datetime = since_datetime.replace(tzinfo=pytz.UTC,
                                                hour=0,
                                                minute=0,
                                                second=0)
        if identifier:
            try:
                since_datetime = dateutil.parser.parse(identifier)
            except Exception as e:
                logger.error("Error parsing datetime from %s", identifier)
        commits = self._github_repo.commits(since_datetime, path=path)
        self.processCommits(commits,
                            commit_started_callback=commit_started_callback,
                            patch_callback=patch_callback,
                            commit_finished_callback=commit_finished_callback)
        if self._last_date:
            return self._last_date.isoformat()
        if since_datetime:
            return since_datetime.isoformat()
        return None

    def processCommits(self, commits, commit_started_callback, patch_callback,
                       commit_finished_callback):
        if commits is None:
            return None

        # Process oldest first
        commits = commits[::-1]
        logger.debug("processing %d commits", len(commits))
        for github_commit in commits:
            logger.debug("sha: %s", github_commit.sha)
            if github_commit.sha:
                github_commit = self._github_repo.commit(github_commit.sha)

                repo_commit = RepoCommit()
                repo_commit.url = github_commit.html_url
                repo_commit.repo_source = self

                if github_commit.date is not None:
                    self._last_date = dateutil.parser.parse(github_commit.date)

                    repo_commit.date = self._last_date
                    self._last_date += datetime.timedelta(seconds=1)
                    repo_commit.identifier = self._last_date.isoformat()

                    repo_commit.committer_email = github_commit.committer_email
                    repo_commit.committer_name = github_commit.committer_name
                    repo_commit.username = github_commit.committer_login
                    repo_commit.message = github_commit.message

                    repo_commit.sha = github_commit.sha
                    commit_started_callback(repo_commit)

                    if github_commit.files:
                        for file_info in github_commit.files:
                            if file_info.get('patch'):
                                filename = committer_username = None
                                diff = DiffParser(file_info['patch'])
                                repo_patch = RepoPatch(repo_commit=repo_commit)
                                repo_patch.diff = diff
                                repo_patch.filename = file_info.get("filename")
                                patch_callback(repo_patch)

                    commit_finished_callback(repo_commit)
                    logger.debug("commit sha: %s processing complete",
                                 github_commit.sha)
        #logger.debug("done")

    def retrieve_file(self, file_path):
        file_content = self._github_repo.get_raw_file(file_path)
        if file_content is None:
            file_content = ''
        return file_content
예제 #4
0
 def __init__(self, creds=None, host='api.github.com', owner='Pardot', repo='pardot'):
     self._last_date = None
     self._last_identifier = None
     self._no_more_requests_until = None
     github_org = GithubOrg(host, owner, creds)
     self._github_repo = GithubRepo(github_org, repo)
예제 #5
0
class GithubSource(RepoSource):
    def __init__(self, creds=None, host='api.github.com', owner='Pardot', repo='pardot'):
        self._last_date = None
        self._last_identifier = None
        self._no_more_requests_until = None
        github_org = GithubOrg(host, owner, creds)
        self._github_repo = GithubRepo(github_org, repo)

    def processSinceIdentifier(self, identifier, commit_started_callback, patch_callback, commit_finished_callback, path=None):
        since_datetime = datetime.datetime.utcnow()
        since_datetime = since_datetime.replace(tzinfo=pytz.UTC, hour=0, minute=0, second=0)
        if identifier:
            try:
                since_datetime = dateutil.parser.parse(identifier)
            except Exception as e:
                logger.error("Error parsing datetime from %s", identifier)
        commits = self._github_repo.commits(since_datetime, path=path)
        self.processCommits(commits, 
                            commit_started_callback=commit_started_callback,
                            patch_callback=patch_callback,
                            commit_finished_callback=commit_finished_callback);
        if self._last_date:
            return self._last_date.isoformat()
        if since_datetime:
            return since_datetime.isoformat()
        return None

    def processCommits(self, commits, commit_started_callback, patch_callback, commit_finished_callback):
        if commits is None:
            return None

        # Process oldest first
        commits = commits[::-1]
        logger.debug("process %d commits", len(commits))
        for github_commit in commits:
            logger.debug("sha: %s", github_commit.sha)
            if github_commit.sha:
                github_commit = self._github_repo.commit(github_commit.sha)

                repo_commit = RepoCommit();
                repo_commit.url = github_commit.html_url
                repo_commit.repo_source = self

                if github_commit.date is not None:
                    self._last_date = dateutil.parser.parse(github_commit.date)

                    repo_commit.date = self._last_date
                    self._last_date += datetime.timedelta(seconds=1)
                    repo_commit.identifier = self._last_date.isoformat()

                    repo_commit.committer_email = github_commit.committer_email
                    repo_commit.committer_name = github_commit.committer_name
                    repo_commit.username = github_commit.committer_login
                    repo_commit.message = github_commit.message
                    
                    repo_commit.sha = github_commit.sha
                    commit_started_callback(repo_commit)

                    if github_commit.files:
                        for file_info in github_commit.files:
                            if file_info.get('patch'):
                                filename = committer_username = None
                                diff = DiffParser(file_info['patch'])
                                repo_patch = RepoPatch(repo_commit=repo_commit)
                                repo_patch.diff = diff
                                repo_patch.filename = file_info.get("filename")
                                patch_callback(repo_patch)

                    commit_finished_callback(repo_commit)
                    logger.debug("batch fof files processed")
        logger.debug("done")