示例#1
0
    def processCommits(self, commits, commit_started_callback, patch_callback,
                       commit_finished_callback):
        if commits is None:
            return None

        # Process oldest first
        commits = commits[::-1]
        logger.debug("processing %d commits", len(commits))
        for github_commit in commits:
            logger.debug("sha: %s", github_commit.sha)
            if github_commit.sha:
                github_commit = self._github_repo.commit(github_commit.sha)

                repo_commit = RepoCommit()
                repo_commit.url = github_commit.html_url
                repo_commit.repo_source = self

                if github_commit.date is not None:
                    self._last_date = dateutil.parser.parse(github_commit.date)

                    repo_commit.date = self._last_date
                    self._last_date += datetime.timedelta(seconds=1)
                    repo_commit.identifier = self._last_date.isoformat()

                    repo_commit.committer_email = github_commit.committer_email
                    repo_commit.committer_name = github_commit.committer_name
                    repo_commit.username = github_commit.committer_login
                    repo_commit.message = github_commit.message

                    repo_commit.sha = github_commit.sha
                    commit_started_callback(repo_commit)

                    if github_commit.files:
                        for file_info in github_commit.files:
                            if file_info.get('patch'):
                                filename = committer_username = None
                                diff = DiffParser(file_info['patch'])
                                repo_patch = RepoPatch(repo_commit=repo_commit)
                                repo_patch.diff = diff
                                repo_patch.filename = file_info.get("filename")
                                patch_callback(repo_patch)

                    commit_finished_callback(repo_commit)
                    logger.debug("commit sha: %s processing complete",
                                 github_commit.sha)
示例#2
0
    def processCommits(self, commits, commit_started_callback, patch_callback, commit_finished_callback):
        if commits is None:
            return None

        # Process oldest first
        commits = commits[::-1]
        logger.debug("process %d commits", len(commits))
        for github_commit in commits:
            logger.debug("sha: %s", github_commit.sha)
            if github_commit.sha:
                github_commit = self._github_repo.commit(github_commit.sha)

                repo_commit = RepoCommit();
                repo_commit.url = github_commit.html_url
                repo_commit.repo_source = self

                if github_commit.date is not None:
                    self._last_date = dateutil.parser.parse(github_commit.date)

                    repo_commit.date = self._last_date
                    self._last_date += datetime.timedelta(seconds=1)
                    repo_commit.identifier = self._last_date.isoformat()

                    repo_commit.committer_email = github_commit.committer_email
                    repo_commit.committer_name = github_commit.committer_name
                    repo_commit.username = github_commit.committer_login
                    repo_commit.message = github_commit.message
                    
                    repo_commit.sha = github_commit.sha
                    commit_started_callback(repo_commit)

                    if github_commit.files:
                        for file_info in github_commit.files:
                            if file_info.get('patch'):
                                filename = committer_username = None
                                diff = DiffParser(file_info['patch'])
                                repo_patch = RepoPatch(repo_commit=repo_commit)
                                repo_patch.diff = diff
                                repo_patch.filename = file_info.get("filename")
                                patch_callback(repo_patch)

                    commit_finished_callback(repo_commit)
                    logger.debug("batch fof files processed")
        logger.debug("done")
示例#3
0
            def process_patches(process_from_change_id,
                                process_to_change_id=-1,
                                patches_processed=0):
                if process_to_change_id == -1:
                    process_to = "#head"
                else:
                    process_to = "@" + str(process_to_change_id)
                changes = p4.run(
                    'changes', '-m', '%s' % (max_patches), '%s@%s,%s' %
                    (directory, process_from_change_id, process_to))
                changes = changes[::-1]
                if len(changes) == 0:
                    return patches_processed
                earliest_change_id = changes[0].get("change")
                try:
                    if earliest_change_id is None or long(
                            earliest_change_id) < long(process_from_change_id):
                        return patches_processed
                    if changes[0]['change'] != process_from_change_id:
                        patches_processed = process_patches(
                            process_from_change_id,
                            str(long(earliest_change_id) - 1),
                            patches_processed)
                except:
                    pass

                logger.info("processing " + str(len(changes)) +
                            " commit from " + process_from_change_id + " to " +
                            process_to)
                for change in changes:
                    try:
                        if patches_processed >= max_patches:
                            return patches_processed
                        patches_processed += 1
                        change_id = change.get("change")
                        if change_id is None:
                            logger.log("Change id was none: " + directory +
                                       search_for)
                            continue
                        change_time = change.get('time')
                        change_datetime = datetime.datetime.utcfromtimestamp(
                            long(change_time))

                        # option to debug a single commit
                        debug_change_id = settings.get_change_id()
                        if not settings.in_production() and debug_change_id:
                            change_id = debug_change_id
                        describes = p4.run('describe', change_id)
                        if describes is None or len(describes) < 1:
                            continue
                        describe = describes[0]
                        if isinstance(describe, basestring):
                            continue
                        user = describe.get("user")
                        description = describe.get("desc")
                        status = describe.get("status")
                        old_change_id = describe.get("oldChange")

                        files = describe.get("depotFile")
                        types = describe.get("type")
                        actions = describe.get("action")
                        digests = describe.get("digest")
                        revs = describe.get("rev")

                        perforce_commit = RepoCommit()
                        perforce_commit.sha = None
                        perforce_commit.identifier = change_id
                        perforce_commit.repo_source = self

                        perforce_commit.url = "http://www.example.com/?change=" + change_id
                        # get the real email of the user
                        email_address = p4.run('users', user)
                        if len(email_address) >= 1:
                            perforce_commit.committer_email = email_address[
                                0].get('Email')

                        perforce_commit.date = change_datetime
                        perforce_commit.committer_login = user
                        perforce_commit.committer_name = user
                        perforce_commit.committer_type = 'user'
                        perforce_commit.files = files
                        perforce_commit.message = description
                        perforce_commit.username = user

                        commit_started_callback(perforce_commit)

                        if files is None:
                            #Todo: we should still record something here, behavioral
                            continue

                        logger.info("processing commit " + change_id +
                                    " in repo " + self.directory)
                        for i in range(len(files)):
                            filename = files[i]

                            # only check added and edited files
                            if "edit" not in actions[
                                    i] and "add" not in actions[i]:
                                continue

                            if old_change_id == None:
                                # fabricate a changelist if a new file is added. (because there is no previous file version to diff against)
                                file_string = self.retrieve_file(filename)
                                diff2 = file_string.split('\n')
                                diff2.insert(0, '')
                                for i in range(len(diff2)):
                                    diff2[i] = '+ ' + diff2[i]
                            else:
                                diff2 = p4.run('diff2',
                                               '-u',
                                               "%s@%s" %
                                               (filename, old_change_id),
                                               "%s@%s" % (filename, change_id),
                                               tagged=False)
                            # diff2 is an array now and well-formatted
                            # change diff2 to a string before passing to diffparser
                            if diff2 is not None and len(diff2) > 1:
                                diffstring = '\n'.join(diff2[1:])
                                diff = DiffParser(diffstring)
                                repo_patch = RepoPatch(
                                    repo_commit=perforce_commit)
                                repo_patch.diff = diff
                                repo_patch.filename = filename
                                patch_callback(repo_patch)

                        self._last_identifier = change_id
                        commit_finished_callback(perforce_commit)
                    except:
                        print "Unexpected error:", sys.exc_info()[0]
                return patches_processed
示例#4
0
            def process_patches(process_from_change_id, process_to_change_id=-1, patches_processed=0):
                if process_to_change_id == -1:
                    process_to = "#head"
                else:
                    process_to = "@" + str(process_to_change_id)
                changes = p4.run( 'changes', '-m','%s' %(max_patches),'%s@%s,%s' % (directory, process_from_change_id, process_to))
                changes = changes[::-1]
                if len(changes) == 0:
                    return patches_processed;
                earliest_change_id = changes[0].get("change")
                try:
                    if earliest_change_id is None or long(earliest_change_id) < long(process_from_change_id):
                        return patches_processed;
                    if changes[0]['change'] != process_from_change_id:
                        patches_processed = process_patches(process_from_change_id, str(long(earliest_change_id)-1), patches_processed)
                except:
                    pass

                logger.info("processing " + str(len(changes)) + " commit from " + process_from_change_id + " to " + process_to)
                for change in changes:
                    try:
                        if patches_processed >= max_patches:
                            return patches_processed
                        patches_processed += 1
                        change_id = change.get("change")
                        if change_id is None:
                            logger.log("Change id was none: " + directory + search_for)
                            continue
                        change_time = change.get('time')
                        change_datetime = datetime.datetime.utcfromtimestamp(long(change_time))

                        # option to debug a single commit
                        debug_change_id = settings.get_change_id()
                        if not settings.in_production() and debug_change_id:
                            change_id = debug_change_id
                        describes = p4.run( 'describe', change_id)
                        if describes is None or len(describes) < 1:
                            continue
                        describe = describes[0]
                        if isinstance(describe, basestring):
                            continue
                        user = describe.get("user")
                        description = describe.get("desc")
                        status = describe.get("status")
                        old_change_id = describe.get("oldChange")

                        files = describe.get("depotFile")
                        types = describe.get("type")
                        actions = describe.get("action")
                        digests = describe.get("digest")
                        revs = describe.get("rev")

                        perforce_commit = RepoCommit()
                        perforce_commit.sha = None
                        perforce_commit.identifier = change_id
                        perforce_commit.repo_source = self

                        perforce_commit.url = "http://www.example.com/?change=" + change_id
                        # get the real email of the user
                        email_address = p4.run('users', user)
                        if len(email_address) >= 1:
                            perforce_commit.committer_email = email_address[0].get('Email')

                        perforce_commit.date = change_datetime
                        perforce_commit.committer_login = user
                        perforce_commit.committer_name = user
                        perforce_commit.committer_type = 'user'
                        perforce_commit.files = files
                        perforce_commit.message = description
                        perforce_commit.username = user

                        commit_started_callback(perforce_commit)

                        if files is None:
                            #Todo: we should still record something here, behavioral
                            continue

                        logger.info("processing commit " + change_id + " in repo " + self.directory)
                        for i in range(len(files)):
                            filename = files[i]

                            # only check added and edited files
                            if "edit" not in actions[i] and "add" not in actions[i]:
                                continue

                            if old_change_id == None:
                                # fabricate a changelist if a new file is added. (because there is no previous file version to diff against)
                                file_string = self.retrieve_file(filename)
                                diff2 = file_string.split('\n')
                                diff2.insert(0, '') 
                                for i in range(len(diff2)):
                                    diff2[i] = '+ ' + diff2[i]
                            else:
                                diff2 = p4.run('diff2','-u',
                                         "%s@%s" % (filename, old_change_id),
                                         "%s@%s" % (filename, change_id),
                                         tagged=False);
                            # diff2 is an array now and well-formatted
                            # change diff2 to a string before passing to diffparser
                            if diff2 is not None and len(diff2) > 1:
                                diffstring = '\n'.join(diff2[1:])
                                diff = DiffParser(diffstring)
                                repo_patch = RepoPatch(repo_commit=perforce_commit)
                                repo_patch.diff = diff
                                repo_patch.filename = filename
                                patch_callback(repo_patch)

                        self._last_identifier = change_id
                        commit_finished_callback(perforce_commit)
                    except:
                        print "Unexpected error:", sys.exc_info()[0]
                return patches_processed