def fetch_projects_in_manifest(self, options):
manifest = self.get_manifest(options)
projects = list()
logger = self.get_logger() # pylint: disable=E1101
pattern = Pattern(options.pattern)
for node in manifest.get_projects():
path = os.path.join(
self.get_absolute_working_dir(options), # pylint: disable=E1101
'%s.git' % node.name)
if not os.path.exists(path):
logger.warning('%s not existed, ignored', path)
continue
elif not pattern.match('p,project', node.name):
logger.debug('%s ignored by the pattern', node.name)
continue
name = '%s%s' % (options.prefix or '',
pattern.replace(
'p,project', node.name, name=node.name))
projects.append(
GitProject(name,
worktree=path,
gitdir=path,
revision=node.revision,
remote='%s/%s' % (options.remote, name),
bare=True,
pattern=pattern,
source=node.name,
copyfiles=node.copyfiles,
linkfiles=node.linkfiles))
return projects
def get_manifest(options, manifest=None, mirror=False):
refsp = manifest
if not refsp and options is not None:
refsp = options.manifest
working_dir = RepoSubcmd.get_absolute_working_dir(options) # pylint: disable=E1101
if not refsp:
repo = GitProject(
'repo-manifest',
worktree=os.path.join(working_dir, '.repo/manifests'))
_, refsp = repo.ls_remote('--get-url')
if not manifest:
manifest = os.path.realpath(
os.path.join(working_dir, '.repo/manifest.xml'))
return Manifest(
filename=manifest,
refspath=os.path.dirname(refsp),
mirror=mirror or (options is not None and options.mirror))
def fetch_projects_in_manifest(options, filename=None):
manifest = RepoSubcmd.get_manifest(options, filename)
projects = list()
logger = RepoSubcmd.get_logger() # pylint: disable=E1101
pattern = RepoSubcmd.get_patterns(options) # pylint: disable=E1101
for node in manifest.get_projects():
if not os.path.exists(node.path) and \
not options.convert_manifest_file:
logger.warning('%s not existed, ignored', node.path)
continue
elif not pattern.match('project', node.name):
logger.warning('%s ignored by the pattern', node.name)
continue
name = '%s%s' % (options.prefix or '',
pattern.replace(
'project', node.name, name=node.name))
project = GitProject(
name,
worktree=os.path.join(
RepoSubcmd.get_absolute_working_dir(options), node.path), # pylint: disable=E1101
remote='%s/%s' % (options.remote, name),
pattern=pattern,
source=node.name,
copyfiles=node.copyfiles,
linkfiles=node.linkfiles)
if project.is_sha1(node.revision) or \
node.revision.startswith('refs/'):
project.revision = '%s' % node.revision
else:
project.revision = '%s/%s' % (node.remote, node.revision)
projects.append(project)
RepoSubcmd.include_project_manifest(options, projects, pattern)
return projects
def include_project_manifest(options, projects, pattern):
if options.include_init_manifest:
projects.append(
GitProject(
None,
gitdir=os.path.join(
RepoSubcmd.get_absolute_working_dir(options), # pylint: disable=E1101
'.repo/manifests.git'),
worktree=os.path.join(
RepoSubcmd.get_absolute_working_dir(options), # pylint: disable=E1101
'.repo/manifests'),
pattern=pattern))
# substitute project name after initialization
projects[-1].update_(
pattern.replace('project',
projects[-1].uri,
name=projects[-1].uri), options.remote)
def execute(self, options, *args, **kws):
SubCommand.execute(self, options, *args, **kws)
RaiseExceptionIfOptionMissed(options.output, 'output is not set')
name, remote = None, options.remote
if options.remote:
ulp = urlparse.urlparse(options.remote)
if ulp.path:
name = ulp.path.strip('/')
remote = '%s://%s' % (ulp.scheme, ulp.hostname)
if ulp.port:
remote += ':%d' % ulp.port
format = options.format and options.format.lower() # pylint: disable=W0622
GitDiffSubcmd.generate_report(
args, GitProject(None, worktree=options.working_dir), name,
options.output, format, options.pattern, remote, options.immediate,
options.gitiles)
def fetch_projects_in_manifest(self, options):
manifest = self.get_manifest(options)
projects = list()
logger = self.get_logger() # pylint: disable=E1101
pattern = Pattern(options.pattern)
for node in manifest.get_projects():
if not os.path.exists(node.path):
logger.warning('%s not existed, ignored', node.path)
continue
elif not pattern.match('p,project', node.name):
logger.debug('%s ignored by the pattern', node.name)
continue
name = '%s%s' % (
options.prefix or '',
pattern.replace('p,project', node.name, name=node.name))
project = GitProject(
name,
worktree=os.path.join(
self.get_absolute_working_dir(options), node.path), # pylint: disable=E1101
remote='%s/%s' % (options.remote, name),
pattern=pattern,
source=node.name,
copyfiles=node.copyfiles,
linkfiles=node.linkfiles)
if project.is_sha1(node.revision) and \
project.rev_existed(node.revision):
project.revision = '%s' % node.revision
else:
project.revision = '%s/%s' % (node.remote, node.revision)
projects.append(project)
return projects
def execute(self, options, *args, **kws):
RaiseExceptionIfOptionMissed(options.output, 'output is not set')
logger = self.get_logger() # pylint: disable=E1101
pattern = GitDiffSubcmd.build_pattern(options.pattern)
format = options.format and options.format.lower() # pylint: disable=W0622
if not os.path.exists(options.output):
os.makedirs(options.output)
with FormattedFile(
os.path.join(options.output, RepoDiffSubcmd.INDEX_HTML),
'Repo Diff', FormattedFile.HTML) as fp:
pdiff = None
if len(args) > 1:
mandiff = RepoSubcmd.get_manifest(options, args[0])
pdiff = dict()
for node in mandiff.get_projects():
pdiff[node.name] = node
manifest = RepoSubcmd.get_manifest(options, args[1])
elif len(args) == 1:
manifest = RepoSubcmd.get_manifest(options, args[0])
else:
manifest = RepoSubcmd.get_manifest(options,
'.repo/manifest.xml')
for node in manifest.get_projects():
if not os.path.exists(node.path):
logger.warning('%s not existed, ignored', node.path)
continue
#elif not pattern.match('p,project', node.name) and \
# not pattern.match('p,project', node.path):
# logger.warning('%s: ignored with pattern', node.name)
# continue
print 'Project %s' % node.name
remote = options.remote
if not remote:
iremote = manifest.get_remote(
node.remote or manifest.get_default().remote)
if iremote:
remote = iremote.review
if remote:
ulp = urlparse.urlparse(remote)
if not ulp.scheme:
remote = 'http://%s' % remote
project = GitProject(node.name,
worktree=os.path.join(
options.working_dir, node.path),
revision=node.revision)
revisions = list()
if pdiff and node.name in pdiff:
revisions.append(pdiff[node.name].revision)
revisions.append(node.revision)
outputdir = os.path.join(options.output, node.name)
GitDiffSubcmd.generate_report(revisions, project, node.name,
outputdir, format,
options.pattern, remote,
options.immediate,
options.gitiles)
order, sha1s = list(), dict()
if options.immediate:
with open(
os.path.join(outputdir,
GitDiffSubcmd.IMMEDIATE_FILE),
'rb') as fim:
for line in fim:
line = line.strip()
if line.startswith('## '):
#fp.section(line[3:])
continue
elif line.startswith('---'):
continue
elif len(line) == 0:
continue
sha1, author, committer, subject = \
line.strip('"').split(' ', 3)
if not pattern.match('e,email', committer):
continue
elif sha1 in sha1s:
continue
order.append(sha1)
sha1s[sha1] = (author, committer, subject)
def _linked_item(fp, dirpath, name):
return fp.item(name, os.path.join(dirpath, name))
column = list()
column.append(
fp.item(node.name, os.path.join(options.output,
node.name)))
column.append(fp.item('(%d)' % len(order)))
if format in ('all', 'html'):
column.append(
_linked_item(fp, outputdir, GitDiffSubcmd.REPORT_HTML))
if format in ('all', 'text'):
column.append(
_linked_item(fp, outputdir, GitDiffSubcmd.REPORT_TEXT))
if options.pattern:
if format in ('all', 'html'):
column.append(
_linked_item(fp, outputdir,
GitDiffSubcmd.FILTER_HTML))
if format in ('all', 'text'):
column.append(
_linked_item(fp, outputdir,
GitDiffSubcmd.FILTER_TEXT))
fp.section(' '.join([str(col) for col in column]))
if order:
with fp.table() as table:
for sha1 in order:
author, committer, subject = sha1s[sha1]
if not remote:
table.row(sha1, author, committer, subject)
continue
if options.gitiles:
sha1a = fp.item(sha1[:20],
'%s#q,%s' % (remote, sha1))
sha1b = fp.item(
sha1[20:], '%s/plugins/gitiles/%s/+/%s^!' %
(remote, node.name, sha1))
table.row(fp.item((sha1a, sha1b), tag='pre'),
author, committer, subject)
else:
link = fp.item(sha1,
'%s#q,%s' % (remote, sha1),
tag='pre')
table.row(link, author, committer, subject)
return True
def execute(self, options, *args, **kws): # pylint: disable=R0915
SubCommand.execute(self, options, option_import=True, *args, **kws)
logger = Logger.get_logger() # pylint: disable=E1101
pkgs, name = list(), None
for pkg in args:
pkgname, revision = _split_name(
pkg, options.pkg_pattern, options.filter_out_chars)
if name and pkgname != name:
logger.warn(
'Warning: pkgname "%s" mismatched "%s"', pkgname, name)
if not revision:
logger.error(
'Error: %s failed to be recognized with revision' % pkg)
else:
name = pkgname
pkgs.append((os.path.realpath(pkg), pkgname, revision))
if len(pkgs) != len(args):
return
if not options.keep_order:
pkgs.sort(_pkg_sort)
if options.show_order or options.verbose > 0:
print 'Effective packages (%d)' % len(pkgs)
print '----------------------------'
for pkg, pkgname, revision in pkgs:
print '%s %-15s %s' % (pkgname, '[v%s]' % revision, pkg)
print
if options.show_order:
return
RaiseExceptionIfOptionMissed(
options.name, "project name (--name) is not set")
RaiseExceptionIfOptionMissed(
options.pkg_pattern, "pkg pattern (--pkg-pattern) is not set")
RaiseExceptionIfOptionMissed(
args, "no files or directories are specified to import")
if not options.tryrun and options.remote:
gerrit = Gerrit(options.remote)
gerrit.create_project(
options.name,
description=options.description or False,
options=options)
branch = options.branch or 'master'
path, _ = os.path.splitext(os.path.basename(options.name))
path = os.path.realpath(path)
if options.offsite and not os.path.exists(path):
os.makedirs(path)
remote = '%s/%s' % (options.remote.rstrip(), options.name) \
if options.remote else None
project = GitProject(
options.name,
worktree=path,
gitdir='%s/.git' % path,
revision=branch,
remote=remote)
ret = project.init_or_download(
branch, single_branch=True, offsite=options.offsite)
if ret != 0:
logger.error('Failed to init the repo %s' % project)
return False
temp = options.temp_directory or tempfile.mkdtemp()
tags = list()
filter_out = list([r'\.git/'])
for fout in options.filter_out or list():
filter_out.extend(fout.split(','))
for pkg, pkgname, revision in pkgs:
workp = pkg
tmpl = dict({
'n': pkgname,
'name': pkgname,
'N': pkgname.upper(),
'NAME': pkgname.upper(),
'v': revision,
'version': revision,
'V': revision.upper(),
'VERSION': revision.upper()})
if options.message_template:
message = options.message_template % tmpl
else:
message = 'Import %s' % (
'%s%s%s' % (
pkgname,
revision and ' %s' % options.version_prefix,
revision))
if options.use_commit_file:
message = _read_file_with_escape(
pkg, options.enable_escape, message)
if os.path.isfile(pkg):
FileUtils.extract_file(pkg, temp)
workp = temp
if options.init_path:
inited = os.path.join(workp, options.init_path)
if os.path.exists(inited):
workp = inited
if options.washed:
if workp != temp:
os.makedirs(temp)
shutil.copytree(workp, temp, symlinks=True)
# wash the directory
washer = FileWasher(
temp, overrideReadOnly=True,
eol=options.washer_eol,
tab=options.washer_tabsize > 0,
trailing=options.washer_trailing)
if washer.wash(temp):
workp = temp
if options.auto_detect:
dname = os.listdir(workp)
while 0 < len(dname) < 2:
workp += '/%s' % dname[0]
dname = os.listdir(workp)
logger.info('Go into %s' % workp)
if options.washed:
diff = FileDiff(project.path, workp, filter_out,
enable_sccs_pattern=options.filter_out_sccs)
if diff.sync(project, logger) > 0:
ret = 0
timestamp = diff.timestamp
else:
timestamp = _timestamp(workp)
FileUtils.rmtree(project.path, ignore_list=(r'^\.git.*',))
FileUtils.copy_files(workp, project.path)
ret = project.add('--all', project.path)
if ret == 0:
args = list()
if options.author:
args.append('--author="%s"' % options.author)
args.append('-m')
args.append(message)
args.append('--date="%s"' % time.ctime(timestamp))
ret = project.commit(*args)
if ret == 0:
if options.version_template:
tags.append(options.version_template % tmpl)
elif options.local:
trefs = self.override_value( # pylint: disable=E1101
options.refs, options.tag_refs) or ''
if trefs:
trefs += '/'
tags.append(
'%s%s%s' % (
trefs, options.version_prefix, revision))
else:
tags.append(
'%s%s' % (options.version_prefix, revision))
ret, _ = project.tag(tags[-1])
if os.path.lexists(temp):
try:
shutil.rmtree(temp)
except OSError, e:
logger.exception(e)
def execute(self, options, *args, **kws): # pylint: disable=R0915
SubCommand.execute(self, options, option_import=True, *args, **kws)
logger = Logger.get_logger() # pylint: disable=E1101
ret, _, pkgs = PkgImportSubcmd.build_packages(options, args, logger)
if not ret:
return
if options.show_order or (options.verbose and options.verbose > 0):
print('Effective packages (%d)' % len(pkgs))
print('----------------------------')
for pkg, pkgname, revision in pkgs:
print('%s %-15s %s' % (pkgname, '[v%s]' % revision, pkg))
print
if options.show_order:
return
RaiseExceptionIfOptionMissed(
options.name, "project name (--name) is not set")
RaiseExceptionIfOptionMissed(
options.remote or options.offsite, 'remote (--remote) is set')
RaiseExceptionIfOptionMissed(
options.pkg_pattern or options.message_template,
'pkg pattern (--pkg-pattern) is not set')
RaiseExceptionIfOptionMissed(
args, "no files or directories are specified to import")
if not options.dryrun and options.remote:
gerrit = Gerrit(options.remote, options)
gerrit.create_project(
options.name,
description=options.description or False,
options=options)
branch = options.branch or 'master'
name, _ = os.path.splitext(os.path.basename(options.name))
path = os.path.join(options.working_dir, name)
if options.offsite and not os.path.exists(path):
os.makedirs(path)
if options.remote:
ulp = urlparse(options.remote)
if not ulp.scheme:
remote = 'git://%s/%s' % (
options.remote.strip('/'), options.name)
else:
remote = '%s/%s' % (options.remote.strip('/'), options.name)
else:
remote = ''
project = GitProject(
options.name,
worktree=path,
gitdir='%s/.git' % path,
revision=branch,
remote=remote)
optgc = options.extra_values(options.extra_option, 'git-clone')
ret = project.init_or_download(
branch, single_branch=True, offsite=options.offsite,
reference=optgc and optgc.reference)
if ret != 0:
logger.error('Failed to init the repo %s' % project)
return False
filters = list()
if options.washed:
filters = list([r'\.git/'])
for fout in options.filter_out or list():
filters.extend(fout.split(','))
opti = Values.build(
detect_root=options.auto_detect,
directory=options.temp_directory,
filter_sccs=options.filter_out_sccs,
filters=filters,
force=options.force,
local=options.local,
imports=True,
prefix=options.version_prefix,
refs=options.refs,
tag_refs=options.tag_refs,
tmpl_escape=options.enable_escape,
tmpl_file=options.use_commit_file,
tmpl_message=options.message_template,
tmpl_version=options.version_template,
washed=options.washed,
extra=options.extra_values(options.extra_option, 'git-commit'))
tags = list()
for pkg, pkgname, revision in pkgs:
workplace = pkg
if options.init_path:
inited = os.path.join(workplace, options.init_path)
if os.path.exists(inited):
workplace = inited
_, ptags = PkgImportSubcmd.do_import(
project, opti, pkgname, workplace, revision, logger=logger)
if ptags:
tags.extend(ptags)
if not ret and not options.local:
# pylint: disable=E1101
# push the branches
if self.override_value(
options.branches, options.all):
ret = project.push_heads(
branch,
self.override_value(
options.refs, options.head_refs),
force=options.force, dryrun=options.dryrun)
# push the tags
if tags and self.override_value(
options.tags, options.all):
optp = Values.build(fullname=True)
ret = project.push_tags(
tags, self.override_value(
options.refs, options.tag_refs),
options=optp, force=options.force, dryrun=options.dryrun)
# pylint: enable=E1101
return ret == 0
def execute(self, options, *args, **kws):
SubCommand.execute(self, options, *args, **kws)
RaiseExceptionIfOptionMissed(options.git or options.offsite,
'git url (--git-url) is not set')
ulp = urlparse(options.name or '')
RaiseExceptionIfOptionMissed(
ulp.scheme or options.remote,
'Neither git name (--name) nor remote (--remote) is set')
logger = self.get_logger() # pylint: disable=E1101
if ulp.scheme:
remote = ulp.hostname
projectname = ulp.path.strip('/')
else:
remote = options.remote
projectname = self.get_name(options) # pylint: disable=E1101
remote = FileUtils.ensure_path(remote,
prefix='git://',
subdir=projectname,
exists=False)
working_dir = self.get_absolute_working_dir(options) # pylint: disable=E1101
project = GitProject(
options.git,
worktree=working_dir,
gitdir=FileUtils.ensure_path(
working_dir, subdir=None if options.bare else '.git'),
revision=options.revision,
remote=remote,
bare=options.bare,
pattern=GitCloneSubcmd.get_patterns(options) # pylint: disable=E1101
)
ret = 0
if not options.offsite:
optgc = options.extra_values(options.extra_option, 'git-clone')
ret = project.download(url=options.git,
bare=options.bare,
reference=optgc and optgc.reference)
if ret != 0:
raise DownloadError('%s: failed to fetch project' % project)
ulp = urlparse(remote)
# creat the project in the remote
if ulp.scheme in ('ssh', 'git'):
if not options.dryrun and options.remote and options.repo_create:
gerrit = Gerrit(options.remote, options)
gerrit.create_project(ulp.path.strip('/'),
description=options.description,
source=options.git,
options=options)
else:
raise ProcessingError('%s: unknown scheme for remote "%s"' %
(project, remote))
self.do_hook( # pylint: disable=E1101
'pre-push',
options,
dryrun=options.dryrun)
optgp = options.extra_values(options.extra_option, 'git-push')
# push the branches
if self.override_value( # pylint: disable=E1101
options.all, options.heads):
optp = Values.build(extra=optgp,
push_all=options.all
or options.revision is None,
fullname=options.keep_name)
res = project.push_heads(
options.revision,
self.override_value( # pylint: disable=E1101
options.refs, options.head_refs),
options.head_pattern,
options=optp,
push_all=options.all or options.revision is None,
fullname=options.keep_name,
force=options.force,
dryrun=options.dryrun)
ret |= res
if res:
logger.error('Failed to push heads')
# push the tags
if self.override_value( # pylint: disable=E1101
options.all, options.tags):
optp = Values.build(extra=optgp, fullname=options.keep_name)
res = project.push_tags(
None if options.all else options.tag,
self.override_value( # pylint: disable=E1101
options.refs, options.tag_refs),
options.tag_pattern,
options=optp,
force=options.force,
dryrun=options.dryrun)
ret |= res
if res:
logger.error('Failed to push tags')
self.do_hook( # pylint: disable=E1101
'post-push',
options,
dryrun=options.dryrun)
return ret