def insert_entities(fn, options): try: with open(fn) as f: data = json.load(f) except: error('failed to load {}'.format(fn)) raise count = process_og(data, options) commit(**db_args(options)) return count
def insert_one(table, columns, values, options, count): insert(table, columns, values, **db_args(options)) count += 1 if count % 100 == 0: info('processed {} ...'.format(count)) if options.commit_every < 2 or count % options.commit_every == 0: if options.commit_every > 100: info('committing ...') commit(**db_args(options)) return count
def getcommit(self, rev): changes = self.changes[rev] return commit(author=changes.author, date=changes.date, desc=changes.summary, parents=self.parents[rev], rev=rev)
def getcommit(self, version): c = self.catfile(version, "commit") # read the commit hash end = c.find("\n\n") message = c[end + 2:] message = self.recode(message) l = c[:end].splitlines() parents = [] author = committer = None for e in l[1:]: n, v = e.split(" ", 1) if n == "author": p = v.split() tm, tz = p[-2:] author = " ".join(p[:-2]) if author[0] == "<": author = author[1:-1] author = self.recode(author) if n == "committer": p = v.split() tm, tz = p[-2:] committer = " ".join(p[:-2]) if committer[0] == "<": committer = committer[1:-1] committer = self.recode(committer) if n == "parent": parents.append(v) if committer and committer != author: message += "\ncommitter: %s\n" % committer tzs, tzh, tzm = tz[-5:-4] + "1", tz[-4:-2], tz[-2:] tz = -int(tzs) * (int(tzh) * 3600 + int(tzm)) date = tm + " " + str(tz) c = commit(parents=parents, date=date, author=author, desc=message, rev=version) return c
def _parse(self): if self.changeset is not None: return self.changeset = {} maxrev = 0 if self.revs: if len(self.revs) > 1: raise util.Abort(_('cvs source does not support specifying ' 'multiple revs')) # TODO: handle tags try: # patchset number? maxrev = int(self.revs[0]) except ValueError: raise util.Abort(_('revision %s is not a patchset number') % self.revs[0]) d = os.getcwd() try: os.chdir(self.path) id = None cache = 'update' if not self.ui.configbool('convert', 'cvsps.cache', True): cache = None db = cvsps.createlog(self.ui, cache=cache) db = cvsps.createchangeset(self.ui, db, fuzz=int(self.ui.config('convert', 'cvsps.fuzz', 60)), mergeto=self.ui.config('convert', 'cvsps.mergeto', None), mergefrom=self.ui.config('convert', 'cvsps.mergefrom', None)) for cs in db: if maxrev and cs.id > maxrev: break id = str(cs.id) cs.author = self.recode(cs.author) self.lastbranch[cs.branch] = id cs.comment = self.recode(cs.comment) if self.ui.configbool('convert', 'localtimezone'): cs.date = makedatetimestamp(cs.date[0]) date = util.datestr(cs.date, '%Y-%m-%d %H:%M:%S %1%2') self.tags.update(dict.fromkeys(cs.tags, id)) files = {} for f in cs.entries: files[f.file] = "%s%s" % ('.'.join([str(x) for x in f.revision]), ['', '(DEAD)'][f.dead]) # add current commit to set c = commit(author=cs.author, date=date, parents=[str(p.id) for p in cs.parents], desc=cs.comment, branch=cs.branch or '') self.changeset[id] = c self.files[id] = files self.heads = self.lastbranch.values() finally: os.chdir(d)
def test_transactional_manager(self): from common import record_action, commit record_action('transactional.transactional_middleware.LoggingTransactionMiddleware', 'start') #self.assert_log('Performed: start') self.transactional_manager.activate_context() self.transactional_manager.enter() self.transactional_manager.managed(True) self.assert_log('Entering transaction management') middleware = self.transactional_manager.middleware['transactional.transactional_middleware.LoggingTransactionMiddleware'] assert middleware.is_managed() self.record_action('foo') self.assertEqual(1, len(middleware.session.actions)) self.transactional_manager.commit() self.assertEqual(0, len(middleware.session.actions)) self.assert_log('Performed: foo', 'commit') self.transactional_manager.rollback() self.transactional_manager.is_managed() self.record_action('level 1') sp = self.transactional_manager.savepoint_enter() self.record_action('level 2') self.transactional_manager.savepoint_commit(sp) self.assert_not_log('Performed: level 1') self.assert_log('Performed: level 2') sp = self.transactional_manager.savepoint_enter() self.record_action('level 3') self.transactional_manager.savepoint_rollback(sp) self.assert_log('Rollbacked: level 3') self.assert_not_log('Performed: level 1') self.transactional_manager.commit_unless_managed() self.assert_not_log('Performed: level 1') self.transactional_manager.commit() self.assert_log('Performed: level 1') self.transactional_manager.rollback_unless_managed() assert record_action('transactional.transactional_middleware.LoggingTransactionMiddleware', 'foo') self.assert_not_log('Performed: foo') commit() self.assert_log('Performed: foo') self.transactional_manager.leave() self.transactional_manager.deactivate_context()
def getcommit(self, rev): elt = self.changes[rev] date = util.strdate(elt.get('local_date'), '%a %b %d %H:%M:%S %Z %Y') desc = elt.findtext('name') + '\n' + elt.findtext('comment', '') return commit(author=elt.get('author'), date=util.datestr(date), desc=desc.strip(), parents=self.parents[rev])
def getcommit(self, rev): certs = self.mtngetcerts(rev) return commit( author=certs["author"], date=util.datestr(util.strdate(certs["date"], "%Y-%m-%dT%H:%M:%S")), desc=certs["changelog"], rev=rev, parents=self.mtnrun("parents", rev).splitlines(), branch=certs["branch"])
def getcommit(self, rev): certs = self.mtngetcerts(rev) return commit(author=certs["author"], date=util.datestr( util.strdate(certs["date"], "%Y-%m-%dT%H:%M:%S")), desc=certs["changelog"], rev=rev, parents=self.mtnrun("parents", rev).splitlines(), branch=certs["branch"])
def getcommit(self, rev): ctx = self.changectx(rev) parents = [hex(p.node()) for p in ctx.parents() if p.node() != nullid] if self.saverev: crev = rev else: crev = None return commit(author=ctx.user(), date=util.datestr(ctx.date()), desc=ctx.description(), rev=crev, parents=parents, branch=ctx.branch(), extra=ctx.extra())
def getcommit(self, rev): elt = self.changes[rev] date = util.strdate(elt.get('local_date'), '%a %b %d %H:%M:%S %Z %Y') desc = elt.findtext('name') + '\n' + elt.findtext('comment', '') # etree can return unicode objects for name, comment, and author, # so recode() is used to ensure str objects are emitted. return commit(author=self.recode(elt.get('author')), date=util.datestr(date), desc=self.recode(desc).strip(), parents=self.parents[rev])
def _commit_on_success(*args, **kw): try: common.enter_transaction_management() common.managed(True) try: res = func(*args, **kw) except: # All exceptions must be handled here (even string ones). common.rollback() raise else: try: common.commit() except: common.rollback() raise return res finally: common.leave_transaction_management()
def getcommit(self, rev): ctx = self.changectx(rev) parents = [p.hex() for p in self.parents(ctx)] crev = rev return commit(author=ctx.user(), date=util.datestr(ctx.date(), '%Y-%m-%d %H:%M:%S %1%2'), desc=ctx.description(), rev=crev, parents=parents, branch=ctx.branch(), extra=ctx.extra(), sortkey=ctx.rev(), saverev=self.saverev, phase=ctx.phase())
def getcommit(self, rev): ctx = self.changectx(rev) parents = [p.hex() for p in self.parents(ctx)] if self.saverev: crev = rev else: crev = None return commit(author=ctx.user(), date=util.datestr(ctx.date()), desc=ctx.description(), rev=crev, parents=parents, branch=ctx.branch(), extra=ctx.extra(), sortkey=ctx.rev())
def getcommit(self, rev): elt = self.changes[rev] date = util.strdate(elt.get("local_date"), "%a %b %d %H:%M:%S %Z %Y") desc = elt.findtext("name") + "\n" + elt.findtext("comment", "") # etree can return unicode objects for name, comment, and author, # so recode() is used to ensure str objects are emitted. return commit( author=self.recode(elt.get("author")), date=util.datestr(date), desc=self.recode(desc).strip(), parents=self.parents[rev], )
def getcommit(self, rev): extra = {} certs = self.mtngetcerts(rev) if certs.get('suspend') == certs["branch"]: extra['close'] = 1 return commit( author=certs["author"], date=util.datestr(util.strdate(certs["date"], "%Y-%m-%dT%H:%M:%S")), desc=certs["changelog"], rev=rev, parents=self.mtnrun("parents", rev).splitlines(), branch=certs["branch"], extra=extra)
def getcommit(self, rev): extra = {} certs = self.mtngetcerts(rev) if certs.get('suspend') == certs["branch"]: extra['close'] = '1' return commit(author=certs["author"], date=util.datestr( util.strdate(certs["date"], "%Y-%m-%dT%H:%M:%S")), desc=certs["changelog"], rev=rev, parents=self.mtnrun("parents", rev).splitlines(), branch=certs["branch"], extra=extra)
def getcommit(self, rev): ctx = self.changectx(rev) parents = [hex(p) for p in self.parents(ctx)] if self.saverev: crev = rev else: crev = None return commit(author=ctx.user(), date=util.datestr(ctx.date()), desc=ctx.description(), rev=crev, parents=parents, branch=ctx.branch(), extra=ctx.extra(), sortkey=ctx.rev())
def getcommit(self, version): rev = self.sourcerepo.get_revision(version) # populate parent id cache if not rev.parent_ids: parents = [] self._parentids[version] = (revision.NULL_REVISION,) else: parents = self._filterghosts(rev.parent_ids) self._parentids[version] = parents return commit(parents=parents, date='%d %d' % (rev.timestamp, -rev.timezone), author=self.recode(rev.committer), # bzr returns bytestrings or unicode, depending on the content desc=self.recode(rev.message), rev=version)
def getcommit(self, version): rev = self.sourcerepo.get_revision(version) # populate parent id cache if not rev.parent_ids: parents = [] self._parentids[version] = (revision.NULL_REVISION, ) else: parents = self._filterghosts(rev.parent_ids) self._parentids[version] = parents return commit( parents=parents, date='%d %d' % (rev.timestamp, -rev.timezone), author=self.recode(rev.committer), # bzr returns bytestrings or unicode, depending on the content desc=self.recode(rev.message), rev=version)
def getcommit(self, version): rev = self.sourcerepo.get_revision(version) # populate parent id cache if not rev.parent_ids: parents = [] self._parentids[version] = (revision.NULL_REVISION,) else: parents = self._filterghosts(rev.parent_ids) self._parentids[version] = parents branch = self.recode(rev.properties.get('branch-nick', u'default')) if branch == 'trunk': branch = 'default' return commit(parents=parents, date='%d %d' % (rev.timestamp, -rev.timezone), author=self.recode(rev.committer), desc=self.recode(rev.message), branch=branch, rev=version)
def getcommit(self, version): rev = self.sourcerepo.get_revision(version) # populate parent id cache if not rev.parent_ids: parents = [] self._parentids[version] = (revision.NULL_REVISION, ) else: parents = self._filterghosts(rev.parent_ids) self._parentids[version] = parents branch = self.recode(rev.properties.get('branch-nick', u'default')) if branch == 'trunk': branch = 'default' return commit(parents=parents, date='%d %d' % (rev.timestamp, -rev.timezone), author=self.recode(rev.committer), desc=self.recode(rev.message), branch=branch, rev=version)
def _parse(self): if self.changeset is not None: return self.changeset = {} maxrev = 0 cmd = self.cmd if self.rev: # TODO: handle tags try: # patchset number? maxrev = int(self.rev) except ValueError: try: # date util.parsedate(self.rev, ['%Y/%m/%d %H:%M:%S']) cmd = '%s -d "1970/01/01 00:00:01" -d "%s"' % (cmd, self.rev) except util.Abort: raise util.Abort( _('revision %s is not a patchset number or date') % self.rev) d = os.getcwd() try: os.chdir(self.path) id = None state = 0 filerevids = {} if self.builtin: # builtin cvsps code self.ui.status(_('using builtin cvsps\n')) cache = 'update' if not self.ui.configbool('convert', 'cvsps.cache', True): cache = None db = cvsps.createlog(self.ui, cache=cache) db = cvsps.createchangeset( self.ui, db, fuzz=int(self.ui.config('convert', 'cvsps.fuzz', 60)), mergeto=self.ui.config('convert', 'cvsps.mergeto', None), mergefrom=self.ui.config('convert', 'cvsps.mergefrom', None)) for cs in db: if maxrev and cs.id > maxrev: break id = str(cs.id) cs.author = self.recode(cs.author) self.lastbranch[cs.branch] = id cs.comment = self.recode(cs.comment) date = util.datestr(cs.date) self.tags.update(dict.fromkeys(cs.tags, id)) files = {} for f in cs.entries: files[f.file] = "%s%s" % ('.'.join( [str(x) for x in f.revision]), ['', '(DEAD)'][f.dead]) # add current commit to set c = commit(author=cs.author, date=date, parents=[str(p.id) for p in cs.parents], desc=cs.comment, branch=cs.branch or '') self.changeset[id] = c self.files[id] = files else: # external cvsps for l in util.popen(cmd): if state == 0: # header if l.startswith("PatchSet"): id = l[9:-2] if maxrev and int(id) > maxrev: # ignore everything state = 3 elif l.startswith("Date:"): date = util.parsedate(l[6:-1], ["%Y/%m/%d %H:%M:%S"]) date = util.datestr(date) elif l.startswith("Branch:"): branch = l[8:-1] self.parent[id] = self.lastbranch.get( branch, 'bad') self.lastbranch[branch] = id elif l.startswith("Ancestor branch:"): ancestor = l[17:-1] # figure out the parent later self.parent[id] = self.lastbranch[ancestor] elif l.startswith("Author:"): author = self.recode(l[8:-1]) elif l.startswith("Tag:") or l.startswith("Tags:"): t = l[l.index(':') + 1:] t = [ut.strip() for ut in t.split(',')] if (len(t) > 1) or (t[0] and (t[0] != "(none)")): self.tags.update(dict.fromkeys(t, id)) elif l.startswith("Log:"): # switch to gathering log state = 1 log = "" elif state == 1: # log if l == "Members: \n": # switch to gathering members files = {} oldrevs = [] log = self.recode(log[:-1]) state = 2 else: # gather log log += l elif state == 2: # members if l == "\n": # start of next entry state = 0 p = [self.parent[id]] if id == "1": p = [] if branch == "HEAD": branch = "" if branch: latest = 0 # the last changeset that contains a base # file is our parent for r in oldrevs: latest = max(filerevids.get(r, 0), latest) if latest: p = [latest] # add current commit to set c = commit(author=author, date=date, parents=p, desc=log, branch=branch) self.changeset[id] = c self.files[id] = files else: colon = l.rfind(':') file = l[1:colon] rev = l[colon + 1:-2] oldrev, rev = rev.split("->") files[file] = rev # save some information for identifying branch points oldrevs.append("%s:%s" % (oldrev, file)) filerevids["%s:%s" % (rev, file)] = id elif state == 3: # swallow all input continue self.heads = self.lastbranch.values() finally: os.chdir(d)
def _parse(self, ui, path): "Prepare list of P4 filenames and revisions to import" ui.status(_('reading p4 views\n')) # read client spec or view if "/" in path: self._parse_view(path) if path.startswith("//") and path.endswith("/..."): views = {path[:-3]:""} else: views = {"//": ""} else: cmd = 'p4 -G client -o %s' % util.shellquote(path) clientspec = marshal.load(util.popen(cmd, mode='rb')) views = {} for client in clientspec: if client.startswith("View"): sview, cview = clientspec[client].split() self._parse_view(sview) if sview.endswith("...") and cview.endswith("..."): sview = sview[:-3] cview = cview[:-3] cview = cview[2:] cview = cview[cview.find("/") + 1:] views[sview] = cview # list of changes that affect our source files self.p4changes = self.p4changes.keys() self.p4changes.sort(key=int) # list with depot pathnames, longest first vieworder = views.keys() vieworder.sort(key=len, reverse=True) # handle revision limiting startrev = self.ui.config('convert', 'p4.startrev', default=0) self.p4changes = [x for x in self.p4changes if ((not startrev or int(x) >= int(startrev)) and (not self.rev or int(x) <= int(self.rev)))] # now read the full changelists to get the list of file revisions ui.status(_('collecting p4 changelists\n')) lastid = None for change in self.p4changes: cmd = "p4 -G describe -s %s" % change stdout = util.popen(cmd, mode='rb') d = marshal.load(stdout) desc = self.recode(d["desc"]) shortdesc = desc.split("\n", 1)[0] t = '%s %s' % (d["change"], repr(shortdesc)[1:-1]) ui.status(util.ellipsis(t, 80) + '\n') if lastid: parents = [lastid] else: parents = [] date = (int(d["time"]), 0) # timezone not set c = commit(author=self.recode(d["user"]), date=util.datestr(date), parents=parents, desc=desc, branch='', extra={"p4": change}) files = [] i = 0 while ("depotFile%d" % i) in d and ("rev%d" % i) in d: oldname = d["depotFile%d" % i] filename = None for v in vieworder: if oldname.startswith(v): filename = views[v] + oldname[len(v):] break if filename: files.append((filename, d["rev%d" % i])) self.depotname[filename] = oldname i += 1 self.changeset[change] = c self.files[change] = files lastid = change if lastid: self.heads = [lastid]
def parselogentry(orig_paths, revnum, author, date, message): """Return the parsed commit object or None, and True if the revision is a branch root. """ self.ui.debug( _("parsing revision %d (%d changes)\n") % (revnum, len(orig_paths))) branched = False rev = self.revid(revnum) # branch log might return entries for a parent we already have if rev in self.commits or revnum < to_revnum: return None, branched parents = [] # check whether this revision is the start of a branch or part # of a branch renaming orig_paths = sorted(orig_paths.iteritems()) root_paths = [(p, e) for p, e in orig_paths if self.module.startswith(p)] if root_paths: path, ent = root_paths[-1] if ent.copyfrom_path: branched = True newpath = ent.copyfrom_path + self.module[len(path):] # ent.copyfrom_rev may not be the actual last revision previd = self.latest(newpath, ent.copyfrom_rev) if previd is not None: prevmodule, prevnum = self.revsplit(previd)[1:] if prevnum >= self.startrev: parents = [previd] self.ui.note( _('found parent of branch %s at %d: %s\n') % (self.module, prevnum, prevmodule)) else: self.ui.debug( _("no copyfrom path, don't know what to do.\n")) paths = [] # filter out unrelated paths for path, ent in orig_paths: if self.getrelpath(path) is None: continue paths.append((path, ent)) # Example SVN datetime. Includes microseconds. # ISO-8601 conformant # '2007-01-04T17:35:00.902377Z' date = util.parsedate(date[:19] + " UTC", ["%Y-%m-%dT%H:%M:%S"]) log = message and self.recode(message) or '' author = author and self.recode(author) or '' try: branch = self.module.split("/")[-1] if branch == 'trunk': branch = '' except IndexError: branch = None cset = commit(author=author, date=util.datestr(date), desc=log, parents=parents, branch=branch, rev=rev) self.commits[rev] = cset # The parents list is *shared* among self.paths and the # commit object. Both will be updated below. self.paths[rev] = (paths, cset.parents) if self.child_cset and not self.child_cset.parents: self.child_cset.parents[:] = [rev] self.child_cset = cset return cset, branched
def _parse(self, ui, path): "Prepare list of P4 filenames and revisions to import" ui.status(_('reading p4 views\n')) # read client spec or view if "/" in path: self._parse_view(path) if path.startswith("//") and path.endswith("/..."): views = {path[:-3]:""} else: views = {"//": ""} else: cmd = 'p4 -G client -o %s' % util.shellquote(path) clientspec = marshal.load(util.popen(cmd, mode='rb')) views = {} for client in clientspec: if client.startswith("View"): sview, cview = clientspec[client].split() self._parse_view(sview) if sview.endswith("...") and cview.endswith("..."): sview = sview[:-3] cview = cview[:-3] cview = cview[2:] cview = cview[cview.find("/") + 1:] views[sview] = cview # list of changes that affect our source files self.p4changes = self.p4changes.keys() self.p4changes.sort(key=int) # list with depot pathnames, longest first vieworder = views.keys() vieworder.sort(key=len, reverse=True) # handle revision limiting startrev = self.ui.config('convert', 'p4.startrev', default=0) self.p4changes = [x for x in self.p4changes if ((not startrev or int(x) >= int(startrev)) and (not self.revs or int(x) <= int(self.revs[0])))] # now read the full changelists to get the list of file revisions ui.status(_('collecting p4 changelists\n')) lastid = None for change in self.p4changes: cmd = "p4 -G describe -s %s" % change stdout = util.popen(cmd, mode='rb') d = marshal.load(stdout) desc = self.recode(d.get("desc", "")) shortdesc = desc.split("\n", 1)[0] t = '%s %s' % (d["change"], repr(shortdesc)[1:-1]) ui.status(util.ellipsis(t, 80) + '\n') if lastid: parents = [lastid] else: parents = [] date = (int(d["time"]), 0) # timezone not set c = commit(author=self.recode(d["user"]), date=util.datestr(date, '%Y-%m-%d %H:%M:%S %1%2'), parents=parents, desc=desc, branch=None, extra={"p4": change}) files = [] copies = {} copiedfiles = [] i = 0 while ("depotFile%d" % i) in d and ("rev%d" % i) in d: oldname = d["depotFile%d" % i] filename = None for v in vieworder: if oldname.lower().startswith(v.lower()): filename = decodefilename(views[v] + oldname[len(v):]) break if filename: files.append((filename, d["rev%d" % i])) self.depotname[filename] = oldname if (d.get("action%d" % i) == "move/add"): copiedfiles.append(filename) self.localname[oldname] = filename i += 1 # Collect information about copied files for filename in copiedfiles: oldname = self.depotname[filename] flcmd = 'p4 -G filelog %s' \ % util.shellquote(oldname) flstdout = util.popen(flcmd, mode='rb') copiedfilename = None for d in loaditer(flstdout): copiedoldname = None i = 0 while ("change%d" % i) in d: if (d["change%d" % i] == change and d["action%d" % i] == "move/add"): j = 0 while ("file%d,%d" % (i, j)) in d: if d["how%d,%d" % (i, j)] == "moved from": copiedoldname = d["file%d,%d" % (i, j)] break j += 1 i += 1 if copiedoldname and copiedoldname in self.localname: copiedfilename = self.localname[copiedoldname] break if copiedfilename: copies[filename] = copiedfilename else: ui.warn(_("cannot find source for copied file: %s@%s\n") % (filename, change)) self.changeset[change] = c self.files[change] = files self.copies[change] = copies lastid = change if lastid: self.heads = [lastid]
def _parse(self, ui, path): "Prepare list of P4 filenames and revisions to import" ui.status(_('reading p4 views\n')) # read client spec or view if "/" in path: self._parse_view(path) if path.startswith("//") and path.endswith("/..."): views = {path[:-3]:""} else: views = {"//": ""} else: cmd = 'p4 -G client -o "%s"' % path clientspec = marshal.load(util.popen(cmd)) views = {} for client in clientspec: if client.startswith("View"): sview, cview = clientspec[client].split() self._parse_view(sview) if sview.endswith("...") and cview.endswith("..."): sview = sview[:-3] cview = cview[:-3] cview = cview[2:] cview = cview[cview.find("/") + 1:] views[sview] = cview # list of changes that affect our source files self.p4changes = self.p4changes.keys() self.p4changes.sort(key=int) # list with depot pathnames, longest first vieworder = views.keys() vieworder.sort(key=len, reverse=True) # handle revision limiting startrev = self.ui.config('convert', 'p4.startrev', default=0) self.p4changes = [x for x in self.p4changes if ((not startrev or int(x) >= int(startrev)) and (not self.rev or int(x) <= int(self.rev)))] # now read the full changelists to get the list of file revisions ui.status(_('collecting p4 changelists\n')) lastid = None for change in self.p4changes: cmd = "p4 -G describe %s" % change stdout = util.popen(cmd) d = marshal.load(stdout) desc = self.recode(d["desc"]) shortdesc = desc.split("\n", 1)[0] t = '%s %s' % (d["change"], repr(shortdesc)[1:-1]) ui.status(util.ellipsis(t, 80) + '\n') if lastid: parents = [lastid] else: parents = [] date = (int(d["time"]), 0) # timezone not set c = commit(author=self.recode(d["user"]), date=util.datestr(date), parents=parents, desc=desc, branch='', extra={"p4": change}) files = [] i = 0 while ("depotFile%d" % i) in d and ("rev%d" % i) in d: oldname = d["depotFile%d" % i] filename = None for v in vieworder: if oldname.startswith(v): filename = views[v] + oldname[len(v):] break if filename: files.append((filename, d["rev%d" % i])) self.depotname[filename] = oldname i += 1 self.changeset[change] = c self.files[change] = files lastid = change if lastid: self.heads = [lastid]
def _parse(self, ui, path): "Prepare list of P4 filenames and revisions to import" ui.status(_('reading p4 views\n')) # read client spec or view if "/" in path: self._parse_view(path) if path.startswith("//") and path.endswith("/..."): views = {path[:-3]: ""} else: views = {"//": ""} else: cmd = 'p4 -G client -o %s' % util.shellquote(path) clientspec = marshal.load(util.popen(cmd, mode='rb')) views = {} for client in clientspec: if client.startswith("View"): sview, cview = clientspec[client].split() self._parse_view(sview) if sview.endswith("...") and cview.endswith("..."): sview = sview[:-3] cview = cview[:-3] cview = cview[2:] cview = cview[cview.find("/") + 1:] views[sview] = cview # list of changes that affect our source files self.p4changes = self.p4changes.keys() self.p4changes.sort(key=int) # list with depot pathnames, longest first vieworder = views.keys() vieworder.sort(key=len, reverse=True) # handle revision limiting startrev = self.ui.config('convert', 'p4.startrev', default=0) self.p4changes = [ x for x in self.p4changes if ((not startrev or int(x) >= int(startrev)) and ( not self.revs or int(x) <= int(self.revs[0]))) ] # now read the full changelists to get the list of file revisions ui.status(_('collecting p4 changelists\n')) lastid = None for change in self.p4changes: cmd = "p4 -G describe -s %s" % change stdout = util.popen(cmd, mode='rb') d = marshal.load(stdout) desc = self.recode(d.get("desc", "")) shortdesc = desc.split("\n", 1)[0] t = '%s %s' % (d["change"], repr(shortdesc)[1:-1]) ui.status(util.ellipsis(t, 80) + '\n') if lastid: parents = [lastid] else: parents = [] date = (int(d["time"]), 0) # timezone not set c = commit(author=self.recode(d["user"]), date=util.datestr(date, '%Y-%m-%d %H:%M:%S %1%2'), parents=parents, desc=desc, branch=None, extra={"p4": change}) files = [] copies = {} copiedfiles = [] i = 0 while ("depotFile%d" % i) in d and ("rev%d" % i) in d: oldname = d["depotFile%d" % i] filename = None for v in vieworder: if oldname.lower().startswith(v.lower()): filename = decodefilename(views[v] + oldname[len(v):]) break if filename: files.append((filename, d["rev%d" % i])) self.depotname[filename] = oldname if (d.get("action%d" % i) == "move/add"): copiedfiles.append(filename) self.localname[oldname] = filename i += 1 # Collect information about copied files for filename in copiedfiles: oldname = self.depotname[filename] flcmd = 'p4 -G filelog %s' \ % util.shellquote(oldname) flstdout = util.popen(flcmd, mode='rb') copiedfilename = None for d in loaditer(flstdout): copiedoldname = None i = 0 while ("change%d" % i) in d: if (d["change%d" % i] == change and d["action%d" % i] == "move/add"): j = 0 while ("file%d,%d" % (i, j)) in d: if d["how%d,%d" % (i, j)] == "moved from": copiedoldname = d["file%d,%d" % (i, j)] break j += 1 i += 1 if copiedoldname and copiedoldname in self.localname: copiedfilename = self.localname[copiedoldname] break if copiedfilename: copies[filename] = copiedfilename else: ui.warn( _("cannot find source for copied file: %s@%s\n") % (filename, change)) self.changeset[change] = c self.files[change] = files self.copies[change] = copies lastid = change if lastid: self.heads = [lastid]
def _parse(self): if self.changeset: return maxrev = 0 cmd = self.cmd if self.rev: # TODO: handle tags try: # patchset number? maxrev = int(self.rev) except ValueError: try: # date util.parsedate(self.rev, ['%Y/%m/%d %H:%M:%S']) cmd = '%s -d "1970/01/01 00:00:01" -d "%s"' % (cmd, self.rev) except util.Abort: raise util.Abort( 'revision %s is not a patchset number or date' % self.rev) d = os.getcwd() try: os.chdir(self.path) id = None state = 0 filerevids = {} for l in util.popen(cmd): if state == 0: # header if l.startswith("PatchSet"): id = l[9:-2] if maxrev and int(id) > maxrev: # ignore everything state = 3 elif l.startswith("Date"): date = util.parsedate(l[6:-1], ["%Y/%m/%d %H:%M:%S"]) date = util.datestr(date) elif l.startswith("Branch"): branch = l[8:-1] self.parent[id] = self.lastbranch.get(branch, 'bad') self.lastbranch[branch] = id elif l.startswith("Ancestor branch"): ancestor = l[17:-1] # figure out the parent later self.parent[id] = self.lastbranch[ancestor] elif l.startswith("Author"): author = self.recode(l[8:-1]) elif l.startswith("Tag:") or l.startswith("Tags:"): t = l[l.index(':') + 1:] t = [ut.strip() for ut in t.split(',')] if (len(t) > 1) or (t[0] and (t[0] != "(none)")): self.tags.update(dict.fromkeys(t, id)) elif l.startswith("Log:"): # switch to gathering log state = 1 log = "" elif state == 1: # log if l == "Members: \n": # switch to gathering members files = {} oldrevs = [] log = self.recode(log[:-1]) state = 2 else: # gather log log += l elif state == 2: # members if l == "\n": # start of next entry state = 0 p = [self.parent[id]] if id == "1": p = [] if branch == "HEAD": branch = "" if branch: latest = None # the last changeset that contains a base # file is our parent for r in oldrevs: latest = max(filerevids.get(r, None), latest) if latest: p = [latest] # add current commit to set c = commit(author=author, date=date, parents=p, desc=log, branch=branch) self.changeset[id] = c self.files[id] = files else: colon = l.rfind(':') file = l[1:colon] rev = l[colon + 1:-2] oldrev, rev = rev.split("->") files[file] = rev # save some information for identifying branch points oldrevs.append("%s:%s" % (oldrev, file)) filerevids["%s:%s" % (rev, file)] = id elif state == 3: # swallow all input continue self.heads = self.lastbranch.values() finally: os.chdir(d)
def parselogentry(orig_paths, revnum, author, date, message): """Return the parsed commit object or None, and True if the revision is a branch root. """ self.ui.debug("parsing revision %d (%d changes)\n" % (revnum, len(orig_paths))) branched = False rev = self.revid(revnum) # branch log might return entries for a parent we already have if rev in self.commits or revnum < to_revnum: return None, branched parents = [] # check whether this revision is the start of a branch or part # of a branch renaming orig_paths = sorted(orig_paths.iteritems()) root_paths = [(p, e) for p, e in orig_paths if self.module.startswith(p)] if root_paths: path, ent = root_paths[-1] if ent.copyfrom_path: branched = True newpath = ent.copyfrom_path + self.module[len(path):] # ent.copyfrom_rev may not be the actual last revision previd = self.latest(newpath, ent.copyfrom_rev) if previd is not None: prevmodule, prevnum = self.revsplit(previd)[1:] if prevnum >= self.startrev: parents = [previd] self.ui.note( _('found parent of branch %s at %d: %s\n') % (self.module, prevnum, prevmodule)) else: self.ui.debug("no copyfrom path, don't know what to do.\n") paths = [] # filter out unrelated paths for path, ent in orig_paths: if self.getrelpath(path) is None: continue paths.append((path, ent)) # Example SVN datetime. Includes microseconds. # ISO-8601 conformant # '2007-01-04T17:35:00.902377Z' date = util.parsedate(date[:19] + " UTC", ["%Y-%m-%dT%H:%M:%S"]) log = message and self.recode(message) or '' author = author and self.recode(author) or '' try: branch = self.module.split("/")[-1] if branch == 'trunk': branch = '' except IndexError: branch = None cset = commit(author=author, date=util.datestr(date), desc=log, parents=parents, branch=branch, rev=rev) self.commits[rev] = cset # The parents list is *shared* among self.paths and the # commit object. Both will be updated below. self.paths[rev] = (paths, cset.parents) if self.child_cset and not self.child_cset.parents: self.child_cset.parents[:] = [rev] self.child_cset = cset return cset, branched
def _parse(self): if self.changeset is not None: return self.changeset = {} maxrev = 0 cmd = self.cmd if self.rev: # TODO: handle tags try: # patchset number? maxrev = int(self.rev) except ValueError: try: # date util.parsedate(self.rev, ['%Y/%m/%d %H:%M:%S']) cmd = '%s -d "1970/01/01 00:00:01" -d "%s"' % (cmd, self.rev) except util.Abort: raise util.Abort(_('revision %s is not a patchset number or date') % self.rev) d = os.getcwd() try: os.chdir(self.path) id = None state = 0 filerevids = {} if self.builtin: # builtin cvsps code self.ui.status(_('using builtin cvsps\n')) cache = 'update' if not self.ui.configbool('convert', 'cvsps.cache', True): cache = None db = cvsps.createlog(self.ui, cache=cache) db = cvsps.createchangeset(self.ui, db, fuzz=int(self.ui.config('convert', 'cvsps.fuzz', 60)), mergeto=self.ui.config('convert', 'cvsps.mergeto', None), mergefrom=self.ui.config('convert', 'cvsps.mergefrom', None)) for cs in db: if maxrev and cs.id>maxrev: break id = str(cs.id) cs.author = self.recode(cs.author) self.lastbranch[cs.branch] = id cs.comment = self.recode(cs.comment) date = util.datestr(cs.date) self.tags.update(dict.fromkeys(cs.tags, id)) files = {} for f in cs.entries: files[f.file] = "%s%s" % ('.'.join([str(x) for x in f.revision]), ['', '(DEAD)'][f.dead]) # add current commit to set c = commit(author=cs.author, date=date, parents=[str(p.id) for p in cs.parents], desc=cs.comment, branch=cs.branch or '') self.changeset[id] = c self.files[id] = files else: # external cvsps for l in util.popen(cmd): if state == 0: # header if l.startswith("PatchSet"): id = l[9:-2] if maxrev and int(id) > maxrev: # ignore everything state = 3 elif l.startswith("Date:"): date = util.parsedate(l[6:-1], ["%Y/%m/%d %H:%M:%S"]) date = util.datestr(date) elif l.startswith("Branch:"): branch = l[8:-1] self.parent[id] = self.lastbranch.get(branch, 'bad') self.lastbranch[branch] = id elif l.startswith("Ancestor branch:"): ancestor = l[17:-1] # figure out the parent later self.parent[id] = self.lastbranch[ancestor] elif l.startswith("Author:"): author = self.recode(l[8:-1]) elif l.startswith("Tag:") or l.startswith("Tags:"): t = l[l.index(':')+1:] t = [ut.strip() for ut in t.split(',')] if (len(t) > 1) or (t[0] and (t[0] != "(none)")): self.tags.update(dict.fromkeys(t, id)) elif l.startswith("Log:"): # switch to gathering log state = 1 log = "" elif state == 1: # log if l == "Members: \n": # switch to gathering members files = {} oldrevs = [] log = self.recode(log[:-1]) state = 2 else: # gather log log += l elif state == 2: # members if l == "\n": # start of next entry state = 0 p = [self.parent[id]] if id == "1": p = [] if branch == "HEAD": branch = "" if branch: latest = 0 # the last changeset that contains a base # file is our parent for r in oldrevs: latest = max(filerevids.get(r, 0), latest) if latest: p = [latest] # add current commit to set c = commit(author=author, date=date, parents=p, desc=log, branch=branch) self.changeset[id] = c self.files[id] = files else: colon = l.rfind(':') file = l[1:colon] rev = l[colon+1:-2] oldrev, rev = rev.split("->") files[file] = rev # save some information for identifying branch points oldrevs.append("%s:%s" % (oldrev, file)) filerevids["%s:%s" % (rev, file)] = id elif state == 3: # swallow all input continue self.heads = self.lastbranch.values() finally: os.chdir(d)
def _parse(self): if self.changeset: return maxrev = 0 cmd = self.cmd if self.rev: # TODO: handle tags try: # patchset number? maxrev = int(self.rev) except ValueError: try: # date util.parsedate(self.rev, ['%Y/%m/%d %H:%M:%S']) cmd = '%s -d "1970/01/01 00:00:01" -d "%s"' % (cmd, self.rev) except util.Abort: raise util.Abort('revision %s is not a patchset number or date' % self.rev) d = os.getcwd() try: os.chdir(self.path) id = None state = 0 filerevids = {} for l in util.popen(cmd): if state == 0: # header if l.startswith("PatchSet"): id = l[9:-2] if maxrev and int(id) > maxrev: # ignore everything state = 3 elif l.startswith("Date"): date = util.parsedate(l[6:-1], ["%Y/%m/%d %H:%M:%S"]) date = util.datestr(date) elif l.startswith("Branch"): branch = l[8:-1] self.parent[id] = self.lastbranch.get(branch, 'bad') self.lastbranch[branch] = id elif l.startswith("Ancestor branch"): ancestor = l[17:-1] # figure out the parent later self.parent[id] = self.lastbranch[ancestor] elif l.startswith("Author"): author = self.recode(l[8:-1]) elif l.startswith("Tag:") or l.startswith("Tags:"): t = l[l.index(':')+1:] t = [ut.strip() for ut in t.split(',')] if (len(t) > 1) or (t[0] and (t[0] != "(none)")): self.tags.update(dict.fromkeys(t, id)) elif l.startswith("Log:"): # switch to gathering log state = 1 log = "" elif state == 1: # log if l == "Members: \n": # switch to gathering members files = {} oldrevs = [] log = self.recode(log[:-1]) state = 2 else: # gather log log += l elif state == 2: # members if l == "\n": # start of next entry state = 0 p = [self.parent[id]] if id == "1": p = [] if branch == "HEAD": branch = "" if branch: latest = None # the last changeset that contains a base # file is our parent for r in oldrevs: latest = max(filerevids.get(r, None), latest) if latest: p = [latest] # add current commit to set c = commit(author=author, date=date, parents=p, desc=log, branch=branch) self.changeset[id] = c self.files[id] = files else: colon = l.rfind(':') file = l[1:colon] rev = l[colon+1:-2] oldrev, rev = rev.split("->") files[file] = rev # save some information for identifying branch points oldrevs.append("%s:%s" % (oldrev, file)) filerevids["%s:%s" % (rev, file)] = id elif state == 3: # swallow all input continue self.heads = self.lastbranch.values() finally: os.chdir(d)
def parselogentry(orig_paths, revnum, author, date, message): """Return the parsed commit object or None, and True if the revision is a branch root. """ self.ui.debug("parsing revision %d (%d changes)\n" % (revnum, len(orig_paths))) branched = False rev = self.revid(revnum) # branch log might return entries for a parent we already have if (rev in self.commits or revnum < to_revnum): return None, branched parents = [] # check whether this revision is the start of a branch or part # of a branch renaming orig_paths = orig_paths.items() orig_paths.sort() root_paths = [(p, e) for p, e in orig_paths if self.module.startswith(p)] if root_paths: path, ent = root_paths[-1] if ent.copyfrom_path: # If dir was moved while one of its file was removed # the log may look like: # A /dir (from /dir:x) # A /dir/a (from /dir/a:y) # A /dir/b (from /dir/b:z) # ... # for all remaining children. # Let's take the highest child element from rev as source. copies = [ (p, e) for p, e in orig_paths[:-1] if isdescendantof(ent.copyfrom_path, e.copyfrom_path) ] fromrev = max([e.copyfrom_rev for p, e in copies] + [ent.copyfrom_rev]) branched = True newpath = ent.copyfrom_path + self.module[len(path):] # ent.copyfrom_rev may not be the actual last revision previd = self.latest(newpath, fromrev) if previd is not None: prevmodule, prevnum = self.revsplit(previd)[1:] if prevnum >= self.startrev: parents = [previd] self.ui.note( 'found parent of branch %s at %d: %s\n' % (self.module, prevnum, prevmodule)) else: self.ui.debug("No copyfrom path, don't know what to do.\n") paths = [] # filter out unrelated paths for path, ent in orig_paths: if self.getrelpath(path) is None: continue paths.append((path, ent)) # Example SVN datetime. Includes microseconds. # ISO-8601 conformant # '2007-01-04T17:35:00.902377Z' date = util.parsedate(date[:19] + " UTC", ["%Y-%m-%dT%H:%M:%S"]) log = message and self.recode(message) or '' author = author and self.recode(author) or '' try: branch = self.module.split("/")[-1] if branch == 'trunk': branch = '' except IndexError: branch = None cset = commit(author=author, date=util.datestr(date), desc=log, parents=parents, branch=branch, rev=rev.encode('utf-8')) self.commits[rev] = cset # The parents list is *shared* among self.paths and the # commit object. Both will be updated below. self.paths[rev] = (paths, cset.parents) if self.child_cset and not self.child_cset.parents: self.child_cset.parents[:] = [rev] self.child_cset = cset return cset, branched