def insert_entities(fn, options):
try:
with open(fn) as f:
data = json.load(f)
except:
error('failed to load {}'.format(fn))
raise
count = process_og(data, options)
commit(**db_args(options))
return count
def insert_one(table, columns, values, options, count):
insert(table, columns, values, **db_args(options))
count += 1
if count % 100 == 0:
info('processed {} ...'.format(count))
if options.commit_every < 2 or count % options.commit_every == 0:
if options.commit_every > 100:
info('committing ...')
commit(**db_args(options))
return count
def getcommit(self, rev):
changes = self.changes[rev]
return commit(author=changes.author,
date=changes.date,
desc=changes.summary,
parents=self.parents[rev],
rev=rev)
def getcommit(self, version):
c = self.catfile(version, "commit") # read the commit hash
end = c.find("\n\n")
message = c[end + 2:]
message = self.recode(message)
l = c[:end].splitlines()
parents = []
author = committer = None
for e in l[1:]:
n, v = e.split(" ", 1)
if n == "author":
p = v.split()
tm, tz = p[-2:]
author = " ".join(p[:-2])
if author[0] == "<": author = author[1:-1]
author = self.recode(author)
if n == "committer":
p = v.split()
tm, tz = p[-2:]
committer = " ".join(p[:-2])
if committer[0] == "<": committer = committer[1:-1]
committer = self.recode(committer)
if n == "parent":
parents.append(v)
if committer and committer != author:
message += "\ncommitter: %s\n" % committer
tzs, tzh, tzm = tz[-5:-4] + "1", tz[-4:-2], tz[-2:]
tz = -int(tzs) * (int(tzh) * 3600 + int(tzm))
date = tm + " " + str(tz)
c = commit(parents=parents, date=date, author=author, desc=message,
rev=version)
return c
def _parse(self):
if self.changeset is not None:
return
self.changeset = {}
maxrev = 0
if self.revs:
if len(self.revs) > 1:
raise util.Abort(_('cvs source does not support specifying '
'multiple revs'))
# TODO: handle tags
try:
# patchset number?
maxrev = int(self.revs[0])
except ValueError:
raise util.Abort(_('revision %s is not a patchset number')
% self.revs[0])
d = os.getcwd()
try:
os.chdir(self.path)
id = None
cache = 'update'
if not self.ui.configbool('convert', 'cvsps.cache', True):
cache = None
db = cvsps.createlog(self.ui, cache=cache)
db = cvsps.createchangeset(self.ui, db,
fuzz=int(self.ui.config('convert', 'cvsps.fuzz', 60)),
mergeto=self.ui.config('convert', 'cvsps.mergeto', None),
mergefrom=self.ui.config('convert', 'cvsps.mergefrom', None))
for cs in db:
if maxrev and cs.id > maxrev:
break
id = str(cs.id)
cs.author = self.recode(cs.author)
self.lastbranch[cs.branch] = id
cs.comment = self.recode(cs.comment)
if self.ui.configbool('convert', 'localtimezone'):
cs.date = makedatetimestamp(cs.date[0])
date = util.datestr(cs.date, '%Y-%m-%d %H:%M:%S %1%2')
self.tags.update(dict.fromkeys(cs.tags, id))
files = {}
for f in cs.entries:
files[f.file] = "%s%s" % ('.'.join([str(x)
for x in f.revision]),
['', '(DEAD)'][f.dead])
# add current commit to set
c = commit(author=cs.author, date=date,
parents=[str(p.id) for p in cs.parents],
desc=cs.comment, branch=cs.branch or '')
self.changeset[id] = c
self.files[id] = files
self.heads = self.lastbranch.values()
finally:
os.chdir(d)
def getcommit(self, version):
c = self.catfile(version, "commit") # read the commit hash
end = c.find("\n\n")
message = c[end + 2:]
message = self.recode(message)
l = c[:end].splitlines()
parents = []
author = committer = None
for e in l[1:]:
n, v = e.split(" ", 1)
if n == "author":
p = v.split()
tm, tz = p[-2:]
author = " ".join(p[:-2])
if author[0] == "<": author = author[1:-1]
author = self.recode(author)
if n == "committer":
p = v.split()
tm, tz = p[-2:]
committer = " ".join(p[:-2])
if committer[0] == "<": committer = committer[1:-1]
committer = self.recode(committer)
if n == "parent":
parents.append(v)
if committer and committer != author:
message += "\ncommitter: %s\n" % committer
tzs, tzh, tzm = tz[-5:-4] + "1", tz[-4:-2], tz[-2:]
tz = -int(tzs) * (int(tzh) * 3600 + int(tzm))
date = tm + " " + str(tz)
c = commit(parents=parents, date=date, author=author, desc=message,
rev=version)
return c
def test_transactional_manager(self):
from common import record_action, commit
record_action('transactional.transactional_middleware.LoggingTransactionMiddleware', 'start')
#self.assert_log('Performed: start')
self.transactional_manager.activate_context()
self.transactional_manager.enter()
self.transactional_manager.managed(True)
self.assert_log('Entering transaction management')
middleware = self.transactional_manager.middleware['transactional.transactional_middleware.LoggingTransactionMiddleware']
assert middleware.is_managed()
self.record_action('foo')
self.assertEqual(1, len(middleware.session.actions))
self.transactional_manager.commit()
self.assertEqual(0, len(middleware.session.actions))
self.assert_log('Performed: foo', 'commit')
self.transactional_manager.rollback()
self.transactional_manager.is_managed()
self.record_action('level 1')
sp = self.transactional_manager.savepoint_enter()
self.record_action('level 2')
self.transactional_manager.savepoint_commit(sp)
self.assert_not_log('Performed: level 1')
self.assert_log('Performed: level 2')
sp = self.transactional_manager.savepoint_enter()
self.record_action('level 3')
self.transactional_manager.savepoint_rollback(sp)
self.assert_log('Rollbacked: level 3')
self.assert_not_log('Performed: level 1')
self.transactional_manager.commit_unless_managed()
self.assert_not_log('Performed: level 1')
self.transactional_manager.commit()
self.assert_log('Performed: level 1')
self.transactional_manager.rollback_unless_managed()
assert record_action('transactional.transactional_middleware.LoggingTransactionMiddleware', 'foo')
self.assert_not_log('Performed: foo')
commit()
self.assert_log('Performed: foo')
self.transactional_manager.leave()
self.transactional_manager.deactivate_context()
def getcommit(self, rev):
elt = self.changes[rev]
date = util.strdate(elt.get('local_date'), '%a %b %d %H:%M:%S %Z %Y')
desc = elt.findtext('name') + '\n' + elt.findtext('comment', '')
return commit(author=elt.get('author'),
date=util.datestr(date),
desc=desc.strip(),
parents=self.parents[rev])
def getcommit(self, rev):
certs = self.mtngetcerts(rev)
return commit(
author=certs["author"],
date=util.datestr(util.strdate(certs["date"], "%Y-%m-%dT%H:%M:%S")),
desc=certs["changelog"],
rev=rev,
parents=self.mtnrun("parents", rev).splitlines(),
branch=certs["branch"])
def getcommit(self, rev):
certs = self.mtngetcerts(rev)
return commit(author=certs["author"],
date=util.datestr(
util.strdate(certs["date"], "%Y-%m-%dT%H:%M:%S")),
desc=certs["changelog"],
rev=rev,
parents=self.mtnrun("parents", rev).splitlines(),
branch=certs["branch"])
def getcommit(self, rev):
ctx = self.changectx(rev)
parents = [hex(p.node()) for p in ctx.parents() if p.node() != nullid]
if self.saverev:
crev = rev
else:
crev = None
return commit(author=ctx.user(), date=util.datestr(ctx.date()),
desc=ctx.description(), rev=crev, parents=parents,
branch=ctx.branch(), extra=ctx.extra())
def getcommit(self, rev):
elt = self.changes[rev]
date = util.strdate(elt.get('local_date'), '%a %b %d %H:%M:%S %Z %Y')
desc = elt.findtext('name') + '\n' + elt.findtext('comment', '')
# etree can return unicode objects for name, comment, and author,
# so recode() is used to ensure str objects are emitted.
return commit(author=self.recode(elt.get('author')),
date=util.datestr(date),
desc=self.recode(desc).strip(),
parents=self.parents[rev])
def getcommit(self, rev):
elt = self.changes[rev]
date = util.strdate(elt.get('local_date'), '%a %b %d %H:%M:%S %Z %Y')
desc = elt.findtext('name') + '\n' + elt.findtext('comment', '')
# etree can return unicode objects for name, comment, and author,
# so recode() is used to ensure str objects are emitted.
return commit(author=self.recode(elt.get('author')),
date=util.datestr(date),
desc=self.recode(desc).strip(),
parents=self.parents[rev])
def _commit_on_success(*args, **kw):
try:
common.enter_transaction_management()
common.managed(True)
try:
res = func(*args, **kw)
except:
# All exceptions must be handled here (even string ones).
common.rollback()
raise
else:
try:
common.commit()
except:
common.rollback()
raise
return res
finally:
common.leave_transaction_management()
def getcommit(self, rev):
ctx = self.changectx(rev)
parents = [p.hex() for p in self.parents(ctx)]
crev = rev
return commit(author=ctx.user(),
date=util.datestr(ctx.date(), '%Y-%m-%d %H:%M:%S %1%2'),
desc=ctx.description(), rev=crev, parents=parents,
branch=ctx.branch(), extra=ctx.extra(),
sortkey=ctx.rev(), saverev=self.saverev,
phase=ctx.phase())
def getcommit(self, rev):
ctx = self.changectx(rev)
parents = [p.hex() for p in self.parents(ctx)]
if self.saverev:
crev = rev
else:
crev = None
return commit(author=ctx.user(), date=util.datestr(ctx.date()),
desc=ctx.description(), rev=crev, parents=parents,
branch=ctx.branch(), extra=ctx.extra(),
sortkey=ctx.rev())
def getcommit(self, rev):
ctx = self.changectx(rev)
parents = [p.hex() for p in self.parents(ctx)]
crev = rev
return commit(author=ctx.user(),
date=util.datestr(ctx.date(), '%Y-%m-%d %H:%M:%S %1%2'),
desc=ctx.description(), rev=crev, parents=parents,
branch=ctx.branch(), extra=ctx.extra(),
sortkey=ctx.rev(), saverev=self.saverev,
phase=ctx.phase())
def getcommit(self, rev):
elt = self.changes[rev]
date = util.strdate(elt.get("local_date"), "%a %b %d %H:%M:%S %Z %Y")
desc = elt.findtext("name") + "\n" + elt.findtext("comment", "")
# etree can return unicode objects for name, comment, and author,
# so recode() is used to ensure str objects are emitted.
return commit(
author=self.recode(elt.get("author")),
date=util.datestr(date),
desc=self.recode(desc).strip(),
parents=self.parents[rev],
)
def getcommit(self, rev):
extra = {}
certs = self.mtngetcerts(rev)
if certs.get('suspend') == certs["branch"]:
extra['close'] = 1
return commit(
author=certs["author"],
date=util.datestr(util.strdate(certs["date"], "%Y-%m-%dT%H:%M:%S")),
desc=certs["changelog"],
rev=rev,
parents=self.mtnrun("parents", rev).splitlines(),
branch=certs["branch"],
extra=extra)
def getcommit(self, rev):
extra = {}
certs = self.mtngetcerts(rev)
if certs.get('suspend') == certs["branch"]:
extra['close'] = '1'
return commit(author=certs["author"],
date=util.datestr(
util.strdate(certs["date"], "%Y-%m-%dT%H:%M:%S")),
desc=certs["changelog"],
rev=rev,
parents=self.mtnrun("parents", rev).splitlines(),
branch=certs["branch"],
extra=extra)
def getcommit(self, rev):
ctx = self.changectx(rev)
parents = [hex(p.node()) for p in ctx.parents() if p.node() != nullid]
if self.saverev:
crev = rev
else:
crev = None
return commit(author=ctx.user(),
date=util.datestr(ctx.date()),
desc=ctx.description(),
rev=crev,
parents=parents,
branch=ctx.branch(),
extra=ctx.extra())
def getcommit(self, rev):
ctx = self.changectx(rev)
parents = [hex(p) for p in self.parents(ctx)]
if self.saverev:
crev = rev
else:
crev = None
return commit(author=ctx.user(),
date=util.datestr(ctx.date()),
desc=ctx.description(),
rev=crev,
parents=parents,
branch=ctx.branch(),
extra=ctx.extra(),
sortkey=ctx.rev())
def getcommit(self, version):
rev = self.sourcerepo.get_revision(version)
# populate parent id cache
if not rev.parent_ids:
parents = []
self._parentids[version] = (revision.NULL_REVISION,)
else:
parents = self._filterghosts(rev.parent_ids)
self._parentids[version] = parents
return commit(parents=parents,
date='%d %d' % (rev.timestamp, -rev.timezone),
author=self.recode(rev.committer),
# bzr returns bytestrings or unicode, depending on the content
desc=self.recode(rev.message),
rev=version)
def getcommit(self, version):
rev = self.sourcerepo.get_revision(version)
# populate parent id cache
if not rev.parent_ids:
parents = []
self._parentids[version] = (revision.NULL_REVISION, )
else:
parents = self._filterghosts(rev.parent_ids)
self._parentids[version] = parents
return commit(
parents=parents,
date='%d %d' % (rev.timestamp, -rev.timezone),
author=self.recode(rev.committer),
# bzr returns bytestrings or unicode, depending on the content
desc=self.recode(rev.message),
rev=version)
def getcommit(self, version):
rev = self.sourcerepo.get_revision(version)
# populate parent id cache
if not rev.parent_ids:
parents = []
self._parentids[version] = (revision.NULL_REVISION,)
else:
parents = self._filterghosts(rev.parent_ids)
self._parentids[version] = parents
branch = self.recode(rev.properties.get('branch-nick', u'default'))
if branch == 'trunk':
branch = 'default'
return commit(parents=parents,
date='%d %d' % (rev.timestamp, -rev.timezone),
author=self.recode(rev.committer),
desc=self.recode(rev.message),
branch=branch,
rev=version)
def getcommit(self, version):
rev = self.sourcerepo.get_revision(version)
# populate parent id cache
if not rev.parent_ids:
parents = []
self._parentids[version] = (revision.NULL_REVISION, )
else:
parents = self._filterghosts(rev.parent_ids)
self._parentids[version] = parents
branch = self.recode(rev.properties.get('branch-nick', u'default'))
if branch == 'trunk':
branch = 'default'
return commit(parents=parents,
date='%d %d' % (rev.timestamp, -rev.timezone),
author=self.recode(rev.committer),
desc=self.recode(rev.message),
branch=branch,
rev=version)
def _parse(self):
if self.changeset is not None:
return
self.changeset = {}
maxrev = 0
cmd = self.cmd
if self.rev:
# TODO: handle tags
try:
# patchset number?
maxrev = int(self.rev)
except ValueError:
try:
# date
util.parsedate(self.rev, ['%Y/%m/%d %H:%M:%S'])
cmd = '%s -d "1970/01/01 00:00:01" -d "%s"' % (cmd,
self.rev)
except util.Abort:
raise util.Abort(
_('revision %s is not a patchset number or date') %
self.rev)
d = os.getcwd()
try:
os.chdir(self.path)
id = None
state = 0
filerevids = {}
if self.builtin:
# builtin cvsps code
self.ui.status(_('using builtin cvsps\n'))
cache = 'update'
if not self.ui.configbool('convert', 'cvsps.cache', True):
cache = None
db = cvsps.createlog(self.ui, cache=cache)
db = cvsps.createchangeset(
self.ui,
db,
fuzz=int(self.ui.config('convert', 'cvsps.fuzz', 60)),
mergeto=self.ui.config('convert', 'cvsps.mergeto', None),
mergefrom=self.ui.config('convert', 'cvsps.mergefrom',
None))
for cs in db:
if maxrev and cs.id > maxrev:
break
id = str(cs.id)
cs.author = self.recode(cs.author)
self.lastbranch[cs.branch] = id
cs.comment = self.recode(cs.comment)
date = util.datestr(cs.date)
self.tags.update(dict.fromkeys(cs.tags, id))
files = {}
for f in cs.entries:
files[f.file] = "%s%s" % ('.'.join(
[str(x)
for x in f.revision]), ['', '(DEAD)'][f.dead])
# add current commit to set
c = commit(author=cs.author,
date=date,
parents=[str(p.id) for p in cs.parents],
desc=cs.comment,
branch=cs.branch or '')
self.changeset[id] = c
self.files[id] = files
else:
# external cvsps
for l in util.popen(cmd):
if state == 0: # header
if l.startswith("PatchSet"):
id = l[9:-2]
if maxrev and int(id) > maxrev:
# ignore everything
state = 3
elif l.startswith("Date:"):
date = util.parsedate(l[6:-1],
["%Y/%m/%d %H:%M:%S"])
date = util.datestr(date)
elif l.startswith("Branch:"):
branch = l[8:-1]
self.parent[id] = self.lastbranch.get(
branch, 'bad')
self.lastbranch[branch] = id
elif l.startswith("Ancestor branch:"):
ancestor = l[17:-1]
# figure out the parent later
self.parent[id] = self.lastbranch[ancestor]
elif l.startswith("Author:"):
author = self.recode(l[8:-1])
elif l.startswith("Tag:") or l.startswith("Tags:"):
t = l[l.index(':') + 1:]
t = [ut.strip() for ut in t.split(',')]
if (len(t) > 1) or (t[0] and (t[0] != "(none)")):
self.tags.update(dict.fromkeys(t, id))
elif l.startswith("Log:"):
# switch to gathering log
state = 1
log = ""
elif state == 1: # log
if l == "Members: \n":
# switch to gathering members
files = {}
oldrevs = []
log = self.recode(log[:-1])
state = 2
else:
# gather log
log += l
elif state == 2: # members
if l == "\n": # start of next entry
state = 0
p = [self.parent[id]]
if id == "1":
p = []
if branch == "HEAD":
branch = ""
if branch:
latest = 0
# the last changeset that contains a base
# file is our parent
for r in oldrevs:
latest = max(filerevids.get(r, 0), latest)
if latest:
p = [latest]
# add current commit to set
c = commit(author=author,
date=date,
parents=p,
desc=log,
branch=branch)
self.changeset[id] = c
self.files[id] = files
else:
colon = l.rfind(':')
file = l[1:colon]
rev = l[colon + 1:-2]
oldrev, rev = rev.split("->")
files[file] = rev
# save some information for identifying branch points
oldrevs.append("%s:%s" % (oldrev, file))
filerevids["%s:%s" % (rev, file)] = id
elif state == 3:
# swallow all input
continue
self.heads = self.lastbranch.values()
finally:
os.chdir(d)
def _parse(self, ui, path):
"Prepare list of P4 filenames and revisions to import"
ui.status(_('reading p4 views\n'))
# read client spec or view
if "/" in path:
self._parse_view(path)
if path.startswith("//") and path.endswith("/..."):
views = {path[:-3]:""}
else:
views = {"//": ""}
else:
cmd = 'p4 -G client -o %s' % util.shellquote(path)
clientspec = marshal.load(util.popen(cmd, mode='rb'))
views = {}
for client in clientspec:
if client.startswith("View"):
sview, cview = clientspec[client].split()
self._parse_view(sview)
if sview.endswith("...") and cview.endswith("..."):
sview = sview[:-3]
cview = cview[:-3]
cview = cview[2:]
cview = cview[cview.find("/") + 1:]
views[sview] = cview
# list of changes that affect our source files
self.p4changes = self.p4changes.keys()
self.p4changes.sort(key=int)
# list with depot pathnames, longest first
vieworder = views.keys()
vieworder.sort(key=len, reverse=True)
# handle revision limiting
startrev = self.ui.config('convert', 'p4.startrev', default=0)
self.p4changes = [x for x in self.p4changes
if ((not startrev or int(x) >= int(startrev)) and
(not self.rev or int(x) <= int(self.rev)))]
# now read the full changelists to get the list of file revisions
ui.status(_('collecting p4 changelists\n'))
lastid = None
for change in self.p4changes:
cmd = "p4 -G describe -s %s" % change
stdout = util.popen(cmd, mode='rb')
d = marshal.load(stdout)
desc = self.recode(d["desc"])
shortdesc = desc.split("\n", 1)[0]
t = '%s %s' % (d["change"], repr(shortdesc)[1:-1])
ui.status(util.ellipsis(t, 80) + '\n')
if lastid:
parents = [lastid]
else:
parents = []
date = (int(d["time"]), 0) # timezone not set
c = commit(author=self.recode(d["user"]), date=util.datestr(date),
parents=parents, desc=desc, branch='',
extra={"p4": change})
files = []
i = 0
while ("depotFile%d" % i) in d and ("rev%d" % i) in d:
oldname = d["depotFile%d" % i]
filename = None
for v in vieworder:
if oldname.startswith(v):
filename = views[v] + oldname[len(v):]
break
if filename:
files.append((filename, d["rev%d" % i]))
self.depotname[filename] = oldname
i += 1
self.changeset[change] = c
self.files[change] = files
lastid = change
if lastid:
self.heads = [lastid]
def parselogentry(orig_paths, revnum, author, date, message):
"""Return the parsed commit object or None, and True if
the revision is a branch root.
"""
self.ui.debug(
_("parsing revision %d (%d changes)\n") %
(revnum, len(orig_paths)))
branched = False
rev = self.revid(revnum)
# branch log might return entries for a parent we already have
if rev in self.commits or revnum < to_revnum:
return None, branched
parents = []
# check whether this revision is the start of a branch or part
# of a branch renaming
orig_paths = sorted(orig_paths.iteritems())
root_paths = [(p, e) for p, e in orig_paths
if self.module.startswith(p)]
if root_paths:
path, ent = root_paths[-1]
if ent.copyfrom_path:
branched = True
newpath = ent.copyfrom_path + self.module[len(path):]
# ent.copyfrom_rev may not be the actual last revision
previd = self.latest(newpath, ent.copyfrom_rev)
if previd is not None:
prevmodule, prevnum = self.revsplit(previd)[1:]
if prevnum >= self.startrev:
parents = [previd]
self.ui.note(
_('found parent of branch %s at %d: %s\n') %
(self.module, prevnum, prevmodule))
else:
self.ui.debug(
_("no copyfrom path, don't know what to do.\n"))
paths = []
# filter out unrelated paths
for path, ent in orig_paths:
if self.getrelpath(path) is None:
continue
paths.append((path, ent))
# Example SVN datetime. Includes microseconds.
# ISO-8601 conformant
# '2007-01-04T17:35:00.902377Z'
date = util.parsedate(date[:19] + " UTC", ["%Y-%m-%dT%H:%M:%S"])
log = message and self.recode(message) or ''
author = author and self.recode(author) or ''
try:
branch = self.module.split("/")[-1]
if branch == 'trunk':
branch = ''
except IndexError:
branch = None
cset = commit(author=author,
date=util.datestr(date),
desc=log,
parents=parents,
branch=branch,
rev=rev)
self.commits[rev] = cset
# The parents list is *shared* among self.paths and the
# commit object. Both will be updated below.
self.paths[rev] = (paths, cset.parents)
if self.child_cset and not self.child_cset.parents:
self.child_cset.parents[:] = [rev]
self.child_cset = cset
return cset, branched
def getcommit(self, rev):
elt = self.changes[rev]
date = util.strdate(elt.get('local_date'), '%a %b %d %H:%M:%S %Z %Y')
desc = elt.findtext('name') + '\n' + elt.findtext('comment', '')
return commit(author=elt.get('author'), date=util.datestr(date),
desc=desc.strip(), parents=self.parents[rev])
def getcommit(self, rev):
changes = self.changes[rev]
return commit(author=changes.author, date=changes.date,
desc=changes.summary, parents=self.parents[rev], rev=rev)
def _parse(self, ui, path):
"Prepare list of P4 filenames and revisions to import"
ui.status(_('reading p4 views\n'))
# read client spec or view
if "/" in path:
self._parse_view(path)
if path.startswith("//") and path.endswith("/..."):
views = {path[:-3]:""}
else:
views = {"//": ""}
else:
cmd = 'p4 -G client -o %s' % util.shellquote(path)
clientspec = marshal.load(util.popen(cmd, mode='rb'))
views = {}
for client in clientspec:
if client.startswith("View"):
sview, cview = clientspec[client].split()
self._parse_view(sview)
if sview.endswith("...") and cview.endswith("..."):
sview = sview[:-3]
cview = cview[:-3]
cview = cview[2:]
cview = cview[cview.find("/") + 1:]
views[sview] = cview
# list of changes that affect our source files
self.p4changes = self.p4changes.keys()
self.p4changes.sort(key=int)
# list with depot pathnames, longest first
vieworder = views.keys()
vieworder.sort(key=len, reverse=True)
# handle revision limiting
startrev = self.ui.config('convert', 'p4.startrev', default=0)
self.p4changes = [x for x in self.p4changes
if ((not startrev or int(x) >= int(startrev)) and
(not self.revs or int(x) <= int(self.revs[0])))]
# now read the full changelists to get the list of file revisions
ui.status(_('collecting p4 changelists\n'))
lastid = None
for change in self.p4changes:
cmd = "p4 -G describe -s %s" % change
stdout = util.popen(cmd, mode='rb')
d = marshal.load(stdout)
desc = self.recode(d.get("desc", ""))
shortdesc = desc.split("\n", 1)[0]
t = '%s %s' % (d["change"], repr(shortdesc)[1:-1])
ui.status(util.ellipsis(t, 80) + '\n')
if lastid:
parents = [lastid]
else:
parents = []
date = (int(d["time"]), 0) # timezone not set
c = commit(author=self.recode(d["user"]),
date=util.datestr(date, '%Y-%m-%d %H:%M:%S %1%2'),
parents=parents, desc=desc, branch=None,
extra={"p4": change})
files = []
copies = {}
copiedfiles = []
i = 0
while ("depotFile%d" % i) in d and ("rev%d" % i) in d:
oldname = d["depotFile%d" % i]
filename = None
for v in vieworder:
if oldname.lower().startswith(v.lower()):
filename = decodefilename(views[v] + oldname[len(v):])
break
if filename:
files.append((filename, d["rev%d" % i]))
self.depotname[filename] = oldname
if (d.get("action%d" % i) == "move/add"):
copiedfiles.append(filename)
self.localname[oldname] = filename
i += 1
# Collect information about copied files
for filename in copiedfiles:
oldname = self.depotname[filename]
flcmd = 'p4 -G filelog %s' \
% util.shellquote(oldname)
flstdout = util.popen(flcmd, mode='rb')
copiedfilename = None
for d in loaditer(flstdout):
copiedoldname = None
i = 0
while ("change%d" % i) in d:
if (d["change%d" % i] == change and
d["action%d" % i] == "move/add"):
j = 0
while ("file%d,%d" % (i, j)) in d:
if d["how%d,%d" % (i, j)] == "moved from":
copiedoldname = d["file%d,%d" % (i, j)]
break
j += 1
i += 1
if copiedoldname and copiedoldname in self.localname:
copiedfilename = self.localname[copiedoldname]
break
if copiedfilename:
copies[filename] = copiedfilename
else:
ui.warn(_("cannot find source for copied file: %s@%s\n")
% (filename, change))
self.changeset[change] = c
self.files[change] = files
self.copies[change] = copies
lastid = change
if lastid:
self.heads = [lastid]
def _parse(self, ui, path):
"Prepare list of P4 filenames and revisions to import"
ui.status(_('reading p4 views\n'))
# read client spec or view
if "/" in path:
self._parse_view(path)
if path.startswith("//") and path.endswith("/..."):
views = {path[:-3]:""}
else:
views = {"//": ""}
else:
cmd = 'p4 -G client -o "%s"' % path
clientspec = marshal.load(util.popen(cmd))
views = {}
for client in clientspec:
if client.startswith("View"):
sview, cview = clientspec[client].split()
self._parse_view(sview)
if sview.endswith("...") and cview.endswith("..."):
sview = sview[:-3]
cview = cview[:-3]
cview = cview[2:]
cview = cview[cview.find("/") + 1:]
views[sview] = cview
# list of changes that affect our source files
self.p4changes = self.p4changes.keys()
self.p4changes.sort(key=int)
# list with depot pathnames, longest first
vieworder = views.keys()
vieworder.sort(key=len, reverse=True)
# handle revision limiting
startrev = self.ui.config('convert', 'p4.startrev', default=0)
self.p4changes = [x for x in self.p4changes
if ((not startrev or int(x) >= int(startrev)) and
(not self.rev or int(x) <= int(self.rev)))]
# now read the full changelists to get the list of file revisions
ui.status(_('collecting p4 changelists\n'))
lastid = None
for change in self.p4changes:
cmd = "p4 -G describe %s" % change
stdout = util.popen(cmd)
d = marshal.load(stdout)
desc = self.recode(d["desc"])
shortdesc = desc.split("\n", 1)[0]
t = '%s %s' % (d["change"], repr(shortdesc)[1:-1])
ui.status(util.ellipsis(t, 80) + '\n')
if lastid:
parents = [lastid]
else:
parents = []
date = (int(d["time"]), 0) # timezone not set
c = commit(author=self.recode(d["user"]), date=util.datestr(date),
parents=parents, desc=desc, branch='', extra={"p4": change})
files = []
i = 0
while ("depotFile%d" % i) in d and ("rev%d" % i) in d:
oldname = d["depotFile%d" % i]
filename = None
for v in vieworder:
if oldname.startswith(v):
filename = views[v] + oldname[len(v):]
break
if filename:
files.append((filename, d["rev%d" % i]))
self.depotname[filename] = oldname
i += 1
self.changeset[change] = c
self.files[change] = files
lastid = change
if lastid:
self.heads = [lastid]
def _parse(self, ui, path):
"Prepare list of P4 filenames and revisions to import"
ui.status(_('reading p4 views\n'))
# read client spec or view
if "/" in path:
self._parse_view(path)
if path.startswith("//") and path.endswith("/..."):
views = {path[:-3]: ""}
else:
views = {"//": ""}
else:
cmd = 'p4 -G client -o %s' % util.shellquote(path)
clientspec = marshal.load(util.popen(cmd, mode='rb'))
views = {}
for client in clientspec:
if client.startswith("View"):
sview, cview = clientspec[client].split()
self._parse_view(sview)
if sview.endswith("...") and cview.endswith("..."):
sview = sview[:-3]
cview = cview[:-3]
cview = cview[2:]
cview = cview[cview.find("/") + 1:]
views[sview] = cview
# list of changes that affect our source files
self.p4changes = self.p4changes.keys()
self.p4changes.sort(key=int)
# list with depot pathnames, longest first
vieworder = views.keys()
vieworder.sort(key=len, reverse=True)
# handle revision limiting
startrev = self.ui.config('convert', 'p4.startrev', default=0)
self.p4changes = [
x for x in self.p4changes
if ((not startrev or int(x) >= int(startrev)) and (
not self.revs or int(x) <= int(self.revs[0])))
]
# now read the full changelists to get the list of file revisions
ui.status(_('collecting p4 changelists\n'))
lastid = None
for change in self.p4changes:
cmd = "p4 -G describe -s %s" % change
stdout = util.popen(cmd, mode='rb')
d = marshal.load(stdout)
desc = self.recode(d.get("desc", ""))
shortdesc = desc.split("\n", 1)[0]
t = '%s %s' % (d["change"], repr(shortdesc)[1:-1])
ui.status(util.ellipsis(t, 80) + '\n')
if lastid:
parents = [lastid]
else:
parents = []
date = (int(d["time"]), 0) # timezone not set
c = commit(author=self.recode(d["user"]),
date=util.datestr(date, '%Y-%m-%d %H:%M:%S %1%2'),
parents=parents,
desc=desc,
branch=None,
extra={"p4": change})
files = []
copies = {}
copiedfiles = []
i = 0
while ("depotFile%d" % i) in d and ("rev%d" % i) in d:
oldname = d["depotFile%d" % i]
filename = None
for v in vieworder:
if oldname.lower().startswith(v.lower()):
filename = decodefilename(views[v] + oldname[len(v):])
break
if filename:
files.append((filename, d["rev%d" % i]))
self.depotname[filename] = oldname
if (d.get("action%d" % i) == "move/add"):
copiedfiles.append(filename)
self.localname[oldname] = filename
i += 1
# Collect information about copied files
for filename in copiedfiles:
oldname = self.depotname[filename]
flcmd = 'p4 -G filelog %s' \
% util.shellquote(oldname)
flstdout = util.popen(flcmd, mode='rb')
copiedfilename = None
for d in loaditer(flstdout):
copiedoldname = None
i = 0
while ("change%d" % i) in d:
if (d["change%d" % i] == change
and d["action%d" % i] == "move/add"):
j = 0
while ("file%d,%d" % (i, j)) in d:
if d["how%d,%d" % (i, j)] == "moved from":
copiedoldname = d["file%d,%d" % (i, j)]
break
j += 1
i += 1
if copiedoldname and copiedoldname in self.localname:
copiedfilename = self.localname[copiedoldname]
break
if copiedfilename:
copies[filename] = copiedfilename
else:
ui.warn(
_("cannot find source for copied file: %s@%s\n") %
(filename, change))
self.changeset[change] = c
self.files[change] = files
self.copies[change] = copies
lastid = change
if lastid:
self.heads = [lastid]
def _parse(self):
if self.changeset:
return
maxrev = 0
cmd = self.cmd
if self.rev:
# TODO: handle tags
try:
# patchset number?
maxrev = int(self.rev)
except ValueError:
try:
# date
util.parsedate(self.rev, ['%Y/%m/%d %H:%M:%S'])
cmd = '%s -d "1970/01/01 00:00:01" -d "%s"' % (cmd,
self.rev)
except util.Abort:
raise util.Abort(
'revision %s is not a patchset number or date' %
self.rev)
d = os.getcwd()
try:
os.chdir(self.path)
id = None
state = 0
filerevids = {}
for l in util.popen(cmd):
if state == 0: # header
if l.startswith("PatchSet"):
id = l[9:-2]
if maxrev and int(id) > maxrev:
# ignore everything
state = 3
elif l.startswith("Date"):
date = util.parsedate(l[6:-1], ["%Y/%m/%d %H:%M:%S"])
date = util.datestr(date)
elif l.startswith("Branch"):
branch = l[8:-1]
self.parent[id] = self.lastbranch.get(branch, 'bad')
self.lastbranch[branch] = id
elif l.startswith("Ancestor branch"):
ancestor = l[17:-1]
# figure out the parent later
self.parent[id] = self.lastbranch[ancestor]
elif l.startswith("Author"):
author = self.recode(l[8:-1])
elif l.startswith("Tag:") or l.startswith("Tags:"):
t = l[l.index(':') + 1:]
t = [ut.strip() for ut in t.split(',')]
if (len(t) > 1) or (t[0] and (t[0] != "(none)")):
self.tags.update(dict.fromkeys(t, id))
elif l.startswith("Log:"):
# switch to gathering log
state = 1
log = ""
elif state == 1: # log
if l == "Members: \n":
# switch to gathering members
files = {}
oldrevs = []
log = self.recode(log[:-1])
state = 2
else:
# gather log
log += l
elif state == 2: # members
if l == "\n": # start of next entry
state = 0
p = [self.parent[id]]
if id == "1":
p = []
if branch == "HEAD":
branch = ""
if branch:
latest = None
# the last changeset that contains a base
# file is our parent
for r in oldrevs:
latest = max(filerevids.get(r, None), latest)
if latest:
p = [latest]
# add current commit to set
c = commit(author=author,
date=date,
parents=p,
desc=log,
branch=branch)
self.changeset[id] = c
self.files[id] = files
else:
colon = l.rfind(':')
file = l[1:colon]
rev = l[colon + 1:-2]
oldrev, rev = rev.split("->")
files[file] = rev
# save some information for identifying branch points
oldrevs.append("%s:%s" % (oldrev, file))
filerevids["%s:%s" % (rev, file)] = id
elif state == 3:
# swallow all input
continue
self.heads = self.lastbranch.values()
finally:
os.chdir(d)
def parselogentry(orig_paths, revnum, author, date, message):
"""Return the parsed commit object or None, and True if
the revision is a branch root.
"""
self.ui.debug("parsing revision %d (%d changes)\n" %
(revnum, len(orig_paths)))
branched = False
rev = self.revid(revnum)
# branch log might return entries for a parent we already have
if rev in self.commits or revnum < to_revnum:
return None, branched
parents = []
# check whether this revision is the start of a branch or part
# of a branch renaming
orig_paths = sorted(orig_paths.iteritems())
root_paths = [(p, e) for p, e in orig_paths
if self.module.startswith(p)]
if root_paths:
path, ent = root_paths[-1]
if ent.copyfrom_path:
branched = True
newpath = ent.copyfrom_path + self.module[len(path):]
# ent.copyfrom_rev may not be the actual last revision
previd = self.latest(newpath, ent.copyfrom_rev)
if previd is not None:
prevmodule, prevnum = self.revsplit(previd)[1:]
if prevnum >= self.startrev:
parents = [previd]
self.ui.note(
_('found parent of branch %s at %d: %s\n') %
(self.module, prevnum, prevmodule))
else:
self.ui.debug("no copyfrom path, don't know what to do.\n")
paths = []
# filter out unrelated paths
for path, ent in orig_paths:
if self.getrelpath(path) is None:
continue
paths.append((path, ent))
# Example SVN datetime. Includes microseconds.
# ISO-8601 conformant
# '2007-01-04T17:35:00.902377Z'
date = util.parsedate(date[:19] + " UTC", ["%Y-%m-%dT%H:%M:%S"])
log = message and self.recode(message) or ''
author = author and self.recode(author) or ''
try:
branch = self.module.split("/")[-1]
if branch == 'trunk':
branch = ''
except IndexError:
branch = None
cset = commit(author=author,
date=util.datestr(date),
desc=log,
parents=parents,
branch=branch,
rev=rev)
self.commits[rev] = cset
# The parents list is *shared* among self.paths and the
# commit object. Both will be updated below.
self.paths[rev] = (paths, cset.parents)
if self.child_cset and not self.child_cset.parents:
self.child_cset.parents[:] = [rev]
self.child_cset = cset
return cset, branched
def _parse(self):
if self.changeset is not None:
return
self.changeset = {}
maxrev = 0
cmd = self.cmd
if self.rev:
# TODO: handle tags
try:
# patchset number?
maxrev = int(self.rev)
except ValueError:
try:
# date
util.parsedate(self.rev, ['%Y/%m/%d %H:%M:%S'])
cmd = '%s -d "1970/01/01 00:00:01" -d "%s"' % (cmd, self.rev)
except util.Abort:
raise util.Abort(_('revision %s is not a patchset number or date') % self.rev)
d = os.getcwd()
try:
os.chdir(self.path)
id = None
state = 0
filerevids = {}
if self.builtin:
# builtin cvsps code
self.ui.status(_('using builtin cvsps\n'))
cache = 'update'
if not self.ui.configbool('convert', 'cvsps.cache', True):
cache = None
db = cvsps.createlog(self.ui, cache=cache)
db = cvsps.createchangeset(self.ui, db,
fuzz=int(self.ui.config('convert', 'cvsps.fuzz', 60)),
mergeto=self.ui.config('convert', 'cvsps.mergeto', None),
mergefrom=self.ui.config('convert', 'cvsps.mergefrom', None))
for cs in db:
if maxrev and cs.id>maxrev:
break
id = str(cs.id)
cs.author = self.recode(cs.author)
self.lastbranch[cs.branch] = id
cs.comment = self.recode(cs.comment)
date = util.datestr(cs.date)
self.tags.update(dict.fromkeys(cs.tags, id))
files = {}
for f in cs.entries:
files[f.file] = "%s%s" % ('.'.join([str(x) for x in f.revision]),
['', '(DEAD)'][f.dead])
# add current commit to set
c = commit(author=cs.author, date=date,
parents=[str(p.id) for p in cs.parents],
desc=cs.comment, branch=cs.branch or '')
self.changeset[id] = c
self.files[id] = files
else:
# external cvsps
for l in util.popen(cmd):
if state == 0: # header
if l.startswith("PatchSet"):
id = l[9:-2]
if maxrev and int(id) > maxrev:
# ignore everything
state = 3
elif l.startswith("Date:"):
date = util.parsedate(l[6:-1], ["%Y/%m/%d %H:%M:%S"])
date = util.datestr(date)
elif l.startswith("Branch:"):
branch = l[8:-1]
self.parent[id] = self.lastbranch.get(branch, 'bad')
self.lastbranch[branch] = id
elif l.startswith("Ancestor branch:"):
ancestor = l[17:-1]
# figure out the parent later
self.parent[id] = self.lastbranch[ancestor]
elif l.startswith("Author:"):
author = self.recode(l[8:-1])
elif l.startswith("Tag:") or l.startswith("Tags:"):
t = l[l.index(':')+1:]
t = [ut.strip() for ut in t.split(',')]
if (len(t) > 1) or (t[0] and (t[0] != "(none)")):
self.tags.update(dict.fromkeys(t, id))
elif l.startswith("Log:"):
# switch to gathering log
state = 1
log = ""
elif state == 1: # log
if l == "Members: \n":
# switch to gathering members
files = {}
oldrevs = []
log = self.recode(log[:-1])
state = 2
else:
# gather log
log += l
elif state == 2: # members
if l == "\n": # start of next entry
state = 0
p = [self.parent[id]]
if id == "1":
p = []
if branch == "HEAD":
branch = ""
if branch:
latest = 0
# the last changeset that contains a base
# file is our parent
for r in oldrevs:
latest = max(filerevids.get(r, 0), latest)
if latest:
p = [latest]
# add current commit to set
c = commit(author=author, date=date, parents=p,
desc=log, branch=branch)
self.changeset[id] = c
self.files[id] = files
else:
colon = l.rfind(':')
file = l[1:colon]
rev = l[colon+1:-2]
oldrev, rev = rev.split("->")
files[file] = rev
# save some information for identifying branch points
oldrevs.append("%s:%s" % (oldrev, file))
filerevids["%s:%s" % (rev, file)] = id
elif state == 3:
# swallow all input
continue
self.heads = self.lastbranch.values()
finally:
os.chdir(d)
def _parse(self):
if self.changeset:
return
maxrev = 0
cmd = self.cmd
if self.rev:
# TODO: handle tags
try:
# patchset number?
maxrev = int(self.rev)
except ValueError:
try:
# date
util.parsedate(self.rev, ['%Y/%m/%d %H:%M:%S'])
cmd = '%s -d "1970/01/01 00:00:01" -d "%s"' % (cmd, self.rev)
except util.Abort:
raise util.Abort('revision %s is not a patchset number or date' % self.rev)
d = os.getcwd()
try:
os.chdir(self.path)
id = None
state = 0
filerevids = {}
for l in util.popen(cmd):
if state == 0: # header
if l.startswith("PatchSet"):
id = l[9:-2]
if maxrev and int(id) > maxrev:
# ignore everything
state = 3
elif l.startswith("Date"):
date = util.parsedate(l[6:-1], ["%Y/%m/%d %H:%M:%S"])
date = util.datestr(date)
elif l.startswith("Branch"):
branch = l[8:-1]
self.parent[id] = self.lastbranch.get(branch, 'bad')
self.lastbranch[branch] = id
elif l.startswith("Ancestor branch"):
ancestor = l[17:-1]
# figure out the parent later
self.parent[id] = self.lastbranch[ancestor]
elif l.startswith("Author"):
author = self.recode(l[8:-1])
elif l.startswith("Tag:") or l.startswith("Tags:"):
t = l[l.index(':')+1:]
t = [ut.strip() for ut in t.split(',')]
if (len(t) > 1) or (t[0] and (t[0] != "(none)")):
self.tags.update(dict.fromkeys(t, id))
elif l.startswith("Log:"):
# switch to gathering log
state = 1
log = ""
elif state == 1: # log
if l == "Members: \n":
# switch to gathering members
files = {}
oldrevs = []
log = self.recode(log[:-1])
state = 2
else:
# gather log
log += l
elif state == 2: # members
if l == "\n": # start of next entry
state = 0
p = [self.parent[id]]
if id == "1":
p = []
if branch == "HEAD":
branch = ""
if branch:
latest = None
# the last changeset that contains a base
# file is our parent
for r in oldrevs:
latest = max(filerevids.get(r, None), latest)
if latest:
p = [latest]
# add current commit to set
c = commit(author=author, date=date, parents=p,
desc=log, branch=branch)
self.changeset[id] = c
self.files[id] = files
else:
colon = l.rfind(':')
file = l[1:colon]
rev = l[colon+1:-2]
oldrev, rev = rev.split("->")
files[file] = rev
# save some information for identifying branch points
oldrevs.append("%s:%s" % (oldrev, file))
filerevids["%s:%s" % (rev, file)] = id
elif state == 3:
# swallow all input
continue
self.heads = self.lastbranch.values()
finally:
os.chdir(d)
def parselogentry(orig_paths, revnum, author, date, message):
"""Return the parsed commit object or None, and True if
the revision is a branch root.
"""
self.ui.debug("parsing revision %d (%d changes)\n" %
(revnum, len(orig_paths)))
branched = False
rev = self.revid(revnum)
# branch log might return entries for a parent we already have
if (rev in self.commits or revnum < to_revnum):
return None, branched
parents = []
# check whether this revision is the start of a branch or part
# of a branch renaming
orig_paths = orig_paths.items()
orig_paths.sort()
root_paths = [(p, e) for p, e in orig_paths
if self.module.startswith(p)]
if root_paths:
path, ent = root_paths[-1]
if ent.copyfrom_path:
# If dir was moved while one of its file was removed
# the log may look like:
# A /dir (from /dir:x)
# A /dir/a (from /dir/a:y)
# A /dir/b (from /dir/b:z)
# ...
# for all remaining children.
# Let's take the highest child element from rev as source.
copies = [
(p, e) for p, e in orig_paths[:-1]
if isdescendantof(ent.copyfrom_path, e.copyfrom_path)
]
fromrev = max([e.copyfrom_rev
for p, e in copies] + [ent.copyfrom_rev])
branched = True
newpath = ent.copyfrom_path + self.module[len(path):]
# ent.copyfrom_rev may not be the actual last revision
previd = self.latest(newpath, fromrev)
if previd is not None:
prevmodule, prevnum = self.revsplit(previd)[1:]
if prevnum >= self.startrev:
parents = [previd]
self.ui.note(
'found parent of branch %s at %d: %s\n' %
(self.module, prevnum, prevmodule))
else:
self.ui.debug("No copyfrom path, don't know what to do.\n")
paths = []
# filter out unrelated paths
for path, ent in orig_paths:
if self.getrelpath(path) is None:
continue
paths.append((path, ent))
# Example SVN datetime. Includes microseconds.
# ISO-8601 conformant
# '2007-01-04T17:35:00.902377Z'
date = util.parsedate(date[:19] + " UTC", ["%Y-%m-%dT%H:%M:%S"])
log = message and self.recode(message) or ''
author = author and self.recode(author) or ''
try:
branch = self.module.split("/")[-1]
if branch == 'trunk':
branch = ''
except IndexError:
branch = None
cset = commit(author=author,
date=util.datestr(date),
desc=log,
parents=parents,
branch=branch,
rev=rev.encode('utf-8'))
self.commits[rev] = cset
# The parents list is *shared* among self.paths and the
# commit object. Both will be updated below.
self.paths[rev] = (paths, cset.parents)
if self.child_cset and not self.child_cset.parents:
self.child_cset.parents[:] = [rev]
self.child_cset = cset
return cset, branched
