def sync(self):
youngest_stored = self.repos.get_youngest_rev_in_cache(self.db)
if youngest_stored != str(self.repos.youngest_rev):
# Need to cache all information for changes since the last
# sync operation.
if youngest_stored is None:
youngest_stored = '0'
# Obtain a list of changes since the last cache sync
from p4trac.repos import _P4ChangesOutputConsumer
output = _P4ChangesOutputConsumer(self.repos._repos)
self.repos._connection.run('changes', '-l', '-s', 'submitted',
'@>%s' % youngest_stored,
output=output)
if output.errors:
from p4trac.repos import PerforceError
raise PerforceError(output.errors)
changes = output.changes
changes.reverse()
# Perform the precaching of the file history for files in these
# changes.
self.repos._repos.precacheFileHistoryForChanges(changes)
# Call on to the default implementation now that we've cached
# enough information to make it run a bit faster.
CachedRepository.sync(self)
def updateCache(self, fromChange):
from perforce import ConnectionDropped
# Update the database in batches of 1000 changes so that we don't
# overload the virtual memory system by trying to store information
# about every change in the repository at once during the initial
# cache population.
batchSize = 1000
lowerBound = fromChange
upperBound = self.repos.youngest_rev + 1
self.log.debug("Updating cache with changes [%i,%i]" % (lowerBound,
upperBound))
try:
while lowerBound < upperBound:
batchUpperBound = min(lowerBound + batchSize, upperBound)
# Get the next batch of changes to cache
from p4trac.repos import _P4ChangesOutputConsumer
output = _P4ChangesOutputConsumer(self.repos._repos)
self.repos._connection.run('changes', '-l', '-s', 'submitted',
'@>=%i,@<%i' % (lowerBound,
batchUpperBound),
output=output)
if output.errors:
from p4trac.repos import PerforceError
raise PerforceError(output.errors)
changes = output.changes
changes.reverse()
# Pre-cache all information about these changes in memory
# before caching in the database. Clear the in-memory cache
# afterwards to save on memory usage.
self.repos._repos.precacheFileInformationForChanges(changes)
self.storeChangesInDB(changes)
self.repos._repos.clearFileInformationCache()
lowerBound += batchSize
except ConnectionDropped, e:
self.log.debug('Rolling back uncommitted cache updates')
self.db.rollback()
raise TracError('Connection to Perforce server lost')
def previous_rev(self, rev):
self.log.debug('previous_rev(%r)' % rev)
if not isinstance(rev, int):
rev = self.short_rev(rev)
if not isinstance(rev, int):
raise NoSuchChangeset(rev)
from p4trac.repos import _P4ChangesOutputConsumer
output = _P4ChangesOutputConsumer(self._repos)
self._connection.run('changes', '-l', '-s', 'submitted',
'-m', '1', '@<%i' % rev, output=output)
if output.errors:
from p4trac.repos import PerforcError
raise PerforcError(output.errors)
if output.changes:
return max(output.changes)
else:
return None
def get_changesets(self, start, stop):
self.log.debug('PerforceRepository.get_changesets(%r,%r)' % (start, stop))
import datetime
start = datetime.datetime.fromtimestamp(start)
stop = datetime.datetime.fromtimestamp(stop)
startDate = start.strftime('%Y/%m/%d:%H:%M:%S')
stopDate = stop.strftime('%Y/%m/%d:%H:%M:%S')
from p4trac.repos import _P4ChangesOutputConsumer
output = _P4ChangesOutputConsumer(self._repos)
self._connection.run('changes', '-l', '-s', 'submitted',
'@>=%s,@<=%s' % (startDate, stopDate),
output=output)
if output.errors:
from p4trac.repos import PerforceError
raise PerforceError(output.errors)
for change in output.changes:
yield self.get_changeset(change)
def previous_rev(self, rev):
self._log.debug("previous_rev(%r)" % rev)
if not isinstance(rev, int):
rev = self.short_rev(rev)
if not isinstance(rev, int):
raise NoSuchChangeset(rev)
from p4trac.repos import _P4ChangesOutputConsumer
output = _P4ChangesOutputConsumer(self._repos)
self._connection.run("changes", "-l", "-s", "submitted", "-m", "1", "@<%i" % rev, output=output)
if output.errors:
from p4trac.repos import PerforcError
raise PerforcError(output.errors)
if output.changes:
return max(output.changes)
else:
return None
def get_changesets(self, start, stop):
self._log.debug("PerforceRepository.get_changesets(%r,%r)" % (start, stop))
import datetime
start = datetime.datetime.fromtimestamp(start)
stop = datetime.datetime.fromtimestamp(stop)
startDate = start.strftime("%Y/%m/%d:%H:%M:%S")
stopDate = stop.strftime("%Y/%m/%d:%H:%M:%S")
from p4trac.repos import _P4ChangesOutputConsumer
output = _P4ChangesOutputConsumer(self._repos)
self._connection.run("changes", "-l", "-s", "submitted", "@>=%s,@<=%s" % (startDate, stopDate), output=output)
if output.errors:
from p4trac.repos import PerforceError
raise PerforceError(output.errors)
for change in output.changes:
yield self.get_changeset(change)
def get_history(self, limit=None):
self._log.debug('PerforceNode.get_history(%r)' % limit)
if self._node.isFile:
# Force population of the filelog history for efficiency
from p4trac.repos import _P4FileLogOutputConsumer
output = _P4FileLogOutputConsumer(self._repos)
if limit is None:
self._repos._connection.run(
'filelog',
'-i', '-l',
self._repos.fromUnicode(self._nodePath.fullPath),
output=output)
else:
self._repos._connection.run(
'filelog',
'-i', '-l',
'-m', str(limit),
self._repos.fromUnicode(self._nodePath.fullPath),
output=output)
from p4trac.repos import NodePath
currentNode = self._node
i = 0
while i < limit and currentNode is not None:
if currentNode.action in [u'add', u'branch', u'import']:
if currentNode.integrations:
nodePath, how = currentNode.integrations[0]
# TODO: Detect whether the copy was really a move
yield (normalisePath(currentNode.nodePath.path),
currentNode.change,
Changeset.COPY)
currentNode = self._repos.getNode(nodePath)
else:
yield(normalisePath(currentNode.nodePath.path),
currentNode.change,
Changeset.ADD)
if currentNode.fileRevision > 1:
# Get the previous revision
nodePath = NodePath(
currentNode.nodePath.path,
'#%i' % (currentNode.fileRevision - 1))
currentNode = self._repos.getNode(nodePath)
else:
currentNode = None
elif currentNode.action in [u'edit', u'integrate']:
nextNode = None
if currentNode.integrations:
nodePath, how = currentNode.integrations[0]
if how == 'copy':
yield (normalisePath(currentNode.nodePath.path),
currentNode.change,
Changeset.COPY)
nextNode = self._repos.getNode(nodePath)
else:
yield (normalisePath(currentNode.nodePath.path),
currentNode.change,
Changeset.EDIT)
else:
yield (normalisePath(currentNode.nodePath.path),
currentNode.change,
Changeset.EDIT)
if nextNode is None:
if currentNode.fileRevision > 1:
currentNode = self._repos.getNode(
NodePath(currentNode.nodePath.path,
'#%i' % (currentNode.fileRevision-1)))
else:
currentNode = None
else:
currentNode = nextNode
elif currentNode.action in [u'delete']:
yield (normalisePath(currentNode.nodePath.path),
currentNode.change,
Changeset.DELETE)
if currentNode.fileRevision > 1:
currentNode = self._repos.getNode(
NodePath(currentNode.nodePath.path,
'#%i' % (currentNode.fileRevision - 1)))
else:
currentNode = None
i += 1
elif self._node.isDirectory:
# List all changelists that have affected this directory
from p4trac.repos import _P4ChangesOutputConsumer
output = _P4ChangesOutputConsumer(self._repos)
if self._nodePath.isRoot:
queryPath = '//...%s' % self._nodePath.rev
else:
queryPath = '%s/...%s' % (self._nodePath.path,
self._nodePath.rev)
if limit is None:
self._repos._connection.run(
'changes',
'-l', '-s', 'submitted',
self._repos.fromUnicode(queryPath),
output=output)
else:
self._repos._connection.run(
'changes',
'-l', '-s', 'submitted',
'-m', str(limit),
self._repos.fromUnicode(queryPath),
output=output)
if output.errors:
raise PerforceError(output.errors)
changes = output.changes
# And describe the contents of those changelists
from p4trac.repos import _P4DescribeOutputConsumer
output = _P4DescribeOutputConsumer(self._repos)
self._repos._connection.run('describe', '-s',
output=output,
*[str(c) for c in changes])
from p4trac.repos import NodePath
for i in xrange(len(changes)):
change = changes[i]
nodePath = NodePath(self._nodePath.path, change)
if i < len(changes)-1:
prevChange = changes[i+1]
else:
prevChange = change-1
prevNodePath = NodePath(self._nodePath.path, prevChange)
node = self._repos.getNode(nodePath)
prevNode = self._repos.getNode(prevNodePath)
if node.isDirectory:
if prevNode.isDirectory:
yield (normalisePath(self._nodePath.path),
change,
Changeset.EDIT)
else:
yield (normalisePath(self._nodePath.path),
change,
Changeset.ADD)
elif prevNode.isDirectory:
yield (normalisePath(self._nodePath.path),
change,
Changeset.DELETE)
else:
raise NoSuchNode(self._nodePath.path, self._nodePath.rev)
def next_rev(self, rev, path=''):
# Finding the next revision is a little more difficult in Perforce
# as we can only ask for the n most recent changes according to a
# given criteria. We query batches of changes using a binary search
# technique so that the number of changes queried is of the order of
# log N where N is the number of changes greater than rev. This way
# it is still fairly efficient if the next change is 1 or 1000 changes
# later.
self._log.debug('next_rev(%r,%r)' % (rev, path))
from p4trac.repos import NodePath
if not path:
path = u'//'
else:
path = NodePath.normalisePath(path)
node = self._repos.getNode(NodePath(path, rev))
if node.isDirectory:
if node.nodePath.isRoot:
# Handle the root path specially since it encompasses all
# changes and so can use the repository's internal cache.
return self._repos.getNextChange(int(rev))
else:
queryPath = u'%s/...' % node.nodePath.path
else:
queryPath = node.nodePath.path
queryPath = self._repos.fromUnicode(queryPath)
self._log.debug(
u'Looing for next_rev after change %i for %s' % (rev, path))
# Perform a binary-search of sorts for the next revision
batchSize = 50
lowerBound = rev + 1
upperBound = self.youngest_rev
while lowerBound <= upperBound:
if lowerBound + batchSize > upperBound:
batchUpperBound = upperBound
else:
middle = (upperBound + lowerBound) / 2
if middle - lowerBound < batchSize:
batchUpperBound = lowerBound + batchSize
else:
batchUpperBound = middle
self._log.debug(
'Looking for changes in range [%i, %i]' % (lowerBound,
batchUpperBound))
from p4trac.repos import _P4ChangesOutputConsumer
output = _P4ChangesOutputConsumer(self._repos)
self._connection.run('changes', '-l', '-s', 'submitted',
'-m', str(batchSize),
'%s@>=%i,@<=%i' % (queryPath,
lowerBound,
batchUpperBound),
output=output)
if output.errors:
from p4trac.repos import PerforcError
raise PerforcError(output.errors)
if output.changes:
lowest = min(output.changes)
assert lowest >= lowerBound
assert lowest <= batchUpperBound
if lowerBound + batchSize >= batchUpperBound:
# There are no earlier changes
self._log.debug('next_rev is %i' % lowest)
return lowest
else:
# There may be another earlier changes but we know it
# can't be any later than lowest.
upperBound = lowest
else:
# Didn't find any changes in (lowerBound, batchUpperBound)
# Try searching from batchUpperBound + 1 onwards
lowerBound = batchUpperBound + 1
return None
def get_history(self, limit=None):
self._log.debug('PerforceNode.get_history(%r %s)' %
(limit, self._nodePath.fullPath))
if self._node.isFile:
# Force population of the filelog history for efficiency
from p4trac.repos import _P4FileLogOutputConsumer
output = _P4FileLogOutputConsumer(self._repos)
if limit is None:
self._repos._connection.run('filelog',
'-i',
'-l',
self._repos.fromUnicode(
self._nodePath.fullPath),
output=output)
else:
self._repos._connection.run('filelog',
'-i',
'-l',
'-m',
str(limit),
self._repos.fromUnicode(
self._nodePath.fullPath),
output=output)
from p4trac.repos import NodePath
currentNode = self._node
i = 0
while i < limit and currentNode is not None:
if currentNode.action in [u'add', u'branch', u'import']:
if currentNode.integrations:
nodePath, how = currentNode.integrations[0]
# TODO: Detect whether the copy was really a move
yield (normalisePath(currentNode.nodePath.path),
currentNode.change, Changeset.COPY)
currentNode = self._repos.getNode(nodePath)
else:
yield (normalisePath(currentNode.nodePath.path),
currentNode.change, Changeset.ADD)
if currentNode.fileRevision > 1:
# Get the previous revision
nodePath = NodePath(
currentNode.nodePath.path,
'#%i' % (currentNode.fileRevision - 1))
currentNode = self._repos.getNode(nodePath)
else:
currentNode = None
elif currentNode.action in [u'edit', u'integrate']:
nextNode = None
if currentNode.integrations:
nodePath, how = currentNode.integrations[0]
if how == 'copy':
yield (normalisePath(currentNode.nodePath.path),
currentNode.change, Changeset.COPY)
nextNode = self._repos.getNode(nodePath)
else:
yield (normalisePath(currentNode.nodePath.path),
currentNode.change, Changeset.EDIT)
else:
yield (normalisePath(currentNode.nodePath.path),
currentNode.change, Changeset.EDIT)
if nextNode is None:
if currentNode.fileRevision > 1:
currentNode = self._repos.getNode(
NodePath(
currentNode.nodePath.path,
'#%i' % (currentNode.fileRevision - 1)))
else:
currentNode = None
else:
currentNode = nextNode
elif currentNode.action in [u'delete']:
yield (normalisePath(currentNode.nodePath.path),
currentNode.change, Changeset.DELETE)
if currentNode.fileRevision > 1:
currentNode = self._repos.getNode(
NodePath(currentNode.nodePath.path,
'#%i' % (currentNode.fileRevision - 1)))
else:
currentNode = None
i += 1
elif self._node.isDirectory:
# List all changelists that have affected this directory
from p4trac.repos import _P4ChangesOutputConsumer
output = _P4ChangesOutputConsumer(self._repos)
if self._nodePath.isRoot:
queryPath = '@<=%s' % self._nodePath.rev[1:]
else:
queryPath = '%s/...@<=%s' % (self._nodePath.path,
self._nodePath.rev[1:])
if limit is None:
self._repos._connection.run('changes',
'-l',
'-s',
'submitted',
self._repos.fromUnicode(queryPath),
output=output)
else:
self._repos._connection.run('changes',
'-l',
'-s',
'submitted',
'-m',
str(limit),
self._repos.fromUnicode(queryPath),
output=output)
if output.errors:
raise PerforceError(output.errors)
changes = output.changes
# And describe the contents of those changelists
from p4trac.repos import _P4DescribeOutputConsumer
output = _P4DescribeOutputConsumer(self._repos)
self._repos._connection.run('describe',
'-s',
output=output,
*[str(c) for c in changes])
from p4trac.repos import NodePath
for i in xrange(len(changes)):
change = changes[i]
nodePath = NodePath(self._nodePath.path, change)
if i < len(changes) - 1:
prevChange = changes[i + 1]
else:
prevChange = change - 1
prevNodePath = NodePath(self._nodePath.path, prevChange)
node = self._repos.getNode(nodePath)
prevNode = self._repos.getNode(prevNodePath)
if node.isDirectory:
if prevNode.isDirectory:
yield (normalisePath(self._nodePath.path), change,
Changeset.EDIT)
else:
yield (normalisePath(self._nodePath.path), change,
Changeset.ADD)
elif prevNode.isDirectory:
yield (normalisePath(self._nodePath.path), change,
Changeset.DELETE)
else:
raise NoSuchNode(self._nodePath.path, self._nodePath.rev)
def next_rev(self, rev, path=''):
# Finding the next revision is a little more difficult in Perforce
# as we can only ask for the n most recent changes according to a
# given criteria. We query batches of changes using a binary search
# technique so that the number of changes queried is of the order of
# log N where N is the number of changes greater than rev. This way
# it is still fairly efficient if the next change is 1 or 1000 changes
# later.
self.log.debug('next_rev(%r,%r)' % (rev, path))
from p4trac.repos import NodePath
if not path:
path = u'//'
else:
path = NodePath.normalisePath(path)
node = self._repos.getNode(NodePath(path, rev))
if node.isDirectory:
if node.nodePath.isRoot:
# Handle the root path specially since it encompasses all
# changes and so can use the repository's internal cache.
return self._repos.getNextChange(int(rev))
else:
queryPath = u'%s/...' % node.nodePath.path
else:
queryPath = node.nodePath.path
queryPath = self._repos.fromUnicode(queryPath)
self.log.debug(u'Looing for next_rev after change %i for %s' %
(rev, path))
# Perform a binary-search of sorts for the next revision
batchSize = 50
lowerBound = rev + 1
upperBound = self.youngest_rev
while lowerBound <= upperBound:
if lowerBound + batchSize > upperBound:
batchUpperBound = upperBound
else:
middle = (upperBound + lowerBound) / 2
if middle - lowerBound < batchSize:
batchUpperBound = lowerBound + batchSize
else:
batchUpperBound = middle
self.log.debug('Looking for changes in range [%i, %i]' %
(lowerBound, batchUpperBound))
from p4trac.repos import _P4ChangesOutputConsumer
output = _P4ChangesOutputConsumer(self._repos)
self._connection.run('changes',
'-l',
'-s',
'submitted',
'-m',
str(batchSize),
'%s@>=%i,@<=%i' %
(queryPath, lowerBound, batchUpperBound),
output=output)
if output.errors:
from p4trac.repos import PerforcError
raise PerforcError(output.errors)
if output.changes:
lowest = min(output.changes)
assert lowest >= lowerBound
assert lowest <= batchUpperBound
if lowerBound + batchSize >= batchUpperBound:
# There are no earlier changes
self.log.debug('next_rev is %i' % lowest)
return lowest
else:
# There may be another earlier changes but we know it
# can't be any later than lowest.
upperBound = lowest
else:
# Didn't find any changes in (lowerBound, batchUpperBound)
# Try searching from batchUpperBound + 1 onwards
lowerBound = batchUpperBound + 1
return None