def fetch_projects_in_manifest(self, options): manifest = self.get_manifest(options) projects = list() logger = self.get_logger() # pylint: disable=E1101 pattern = Pattern(options.pattern) for node in manifest.get_projects(): path = os.path.join( self.get_absolute_working_dir(options), # pylint: disable=E1101 '%s.git' % node.name) if not os.path.exists(path): logger.warning('%s not existed, ignored', path) continue elif not pattern.match('p,project', node.name): logger.debug('%s ignored by the pattern', node.name) continue name = '%s%s' % (options.prefix or '', pattern.replace( 'p,project', node.name, name=node.name)) projects.append( GitProject(name, worktree=path, gitdir=path, revision=node.revision, remote='%s/%s' % (options.remote, name), bare=True, pattern=pattern, source=node.name, copyfiles=node.copyfiles, linkfiles=node.linkfiles)) return projects
def get_manifest(options, manifest=None, mirror=False): refsp = manifest if not refsp and options is not None: refsp = options.manifest working_dir = RepoSubcmd.get_absolute_working_dir(options) # pylint: disable=E1101 if not refsp: repo = GitProject( 'repo-manifest', worktree=os.path.join(working_dir, '.repo/manifests')) _, refsp = repo.ls_remote('--get-url') if not manifest: manifest = os.path.realpath( os.path.join(working_dir, '.repo/manifest.xml')) return Manifest( filename=manifest, refspath=os.path.dirname(refsp), mirror=mirror or (options is not None and options.mirror))
def fetch_projects_in_manifest(options, filename=None): manifest = RepoSubcmd.get_manifest(options, filename) projects = list() logger = RepoSubcmd.get_logger() # pylint: disable=E1101 pattern = RepoSubcmd.get_patterns(options) # pylint: disable=E1101 for node in manifest.get_projects(): if not os.path.exists(node.path) and \ not options.convert_manifest_file: logger.warning('%s not existed, ignored', node.path) continue elif not pattern.match('project', node.name): logger.warning('%s ignored by the pattern', node.name) continue name = '%s%s' % (options.prefix or '', pattern.replace( 'project', node.name, name=node.name)) project = GitProject( name, worktree=os.path.join( RepoSubcmd.get_absolute_working_dir(options), node.path), # pylint: disable=E1101 remote='%s/%s' % (options.remote, name), pattern=pattern, source=node.name, copyfiles=node.copyfiles, linkfiles=node.linkfiles) if project.is_sha1(node.revision) or \ node.revision.startswith('refs/'): project.revision = '%s' % node.revision else: project.revision = '%s/%s' % (node.remote, node.revision) projects.append(project) RepoSubcmd.include_project_manifest(options, projects, pattern) return projects
def include_project_manifest(options, projects, pattern): if options.include_init_manifest: projects.append( GitProject( None, gitdir=os.path.join( RepoSubcmd.get_absolute_working_dir(options), # pylint: disable=E1101 '.repo/manifests.git'), worktree=os.path.join( RepoSubcmd.get_absolute_working_dir(options), # pylint: disable=E1101 '.repo/manifests'), pattern=pattern)) # substitute project name after initialization projects[-1].update_( pattern.replace('project', projects[-1].uri, name=projects[-1].uri), options.remote)
def execute(self, options, *args, **kws): SubCommand.execute(self, options, *args, **kws) RaiseExceptionIfOptionMissed(options.output, 'output is not set') name, remote = None, options.remote if options.remote: ulp = urlparse.urlparse(options.remote) if ulp.path: name = ulp.path.strip('/') remote = '%s://%s' % (ulp.scheme, ulp.hostname) if ulp.port: remote += ':%d' % ulp.port format = options.format and options.format.lower() # pylint: disable=W0622 GitDiffSubcmd.generate_report( args, GitProject(None, worktree=options.working_dir), name, options.output, format, options.pattern, remote, options.immediate, options.gitiles)
def fetch_projects_in_manifest(self, options): manifest = self.get_manifest(options) projects = list() logger = self.get_logger() # pylint: disable=E1101 pattern = Pattern(options.pattern) for node in manifest.get_projects(): if not os.path.exists(node.path): logger.warning('%s not existed, ignored', node.path) continue elif not pattern.match('p,project', node.name): logger.debug('%s ignored by the pattern', node.name) continue name = '%s%s' % ( options.prefix or '', pattern.replace('p,project', node.name, name=node.name)) project = GitProject( name, worktree=os.path.join( self.get_absolute_working_dir(options), node.path), # pylint: disable=E1101 remote='%s/%s' % (options.remote, name), pattern=pattern, source=node.name, copyfiles=node.copyfiles, linkfiles=node.linkfiles) if project.is_sha1(node.revision) and \ project.rev_existed(node.revision): project.revision = '%s' % node.revision else: project.revision = '%s/%s' % (node.remote, node.revision) projects.append(project) return projects
def execute(self, options, *args, **kws): RaiseExceptionIfOptionMissed(options.output, 'output is not set') logger = self.get_logger() # pylint: disable=E1101 pattern = GitDiffSubcmd.build_pattern(options.pattern) format = options.format and options.format.lower() # pylint: disable=W0622 if not os.path.exists(options.output): os.makedirs(options.output) with FormattedFile( os.path.join(options.output, RepoDiffSubcmd.INDEX_HTML), 'Repo Diff', FormattedFile.HTML) as fp: pdiff = None if len(args) > 1: mandiff = RepoSubcmd.get_manifest(options, args[0]) pdiff = dict() for node in mandiff.get_projects(): pdiff[node.name] = node manifest = RepoSubcmd.get_manifest(options, args[1]) elif len(args) == 1: manifest = RepoSubcmd.get_manifest(options, args[0]) else: manifest = RepoSubcmd.get_manifest(options, '.repo/manifest.xml') for node in manifest.get_projects(): if not os.path.exists(node.path): logger.warning('%s not existed, ignored', node.path) continue #elif not pattern.match('p,project', node.name) and \ # not pattern.match('p,project', node.path): # logger.warning('%s: ignored with pattern', node.name) # continue print 'Project %s' % node.name remote = options.remote if not remote: iremote = manifest.get_remote( node.remote or manifest.get_default().remote) if iremote: remote = iremote.review if remote: ulp = urlparse.urlparse(remote) if not ulp.scheme: remote = 'http://%s' % remote project = GitProject(node.name, worktree=os.path.join( options.working_dir, node.path), revision=node.revision) revisions = list() if pdiff and node.name in pdiff: revisions.append(pdiff[node.name].revision) revisions.append(node.revision) outputdir = os.path.join(options.output, node.name) GitDiffSubcmd.generate_report(revisions, project, node.name, outputdir, format, options.pattern, remote, options.immediate, options.gitiles) order, sha1s = list(), dict() if options.immediate: with open( os.path.join(outputdir, GitDiffSubcmd.IMMEDIATE_FILE), 'rb') as fim: for line in fim: line = line.strip() if line.startswith('## '): #fp.section(line[3:]) continue elif line.startswith('---'): continue elif len(line) == 0: continue sha1, author, committer, subject = \ line.strip('"').split(' ', 3) if not pattern.match('e,email', committer): continue elif sha1 in sha1s: continue order.append(sha1) sha1s[sha1] = (author, committer, subject) def _linked_item(fp, dirpath, name): return fp.item(name, os.path.join(dirpath, name)) column = list() column.append( fp.item(node.name, os.path.join(options.output, node.name))) column.append(fp.item('(%d)' % len(order))) if format in ('all', 'html'): column.append( _linked_item(fp, outputdir, GitDiffSubcmd.REPORT_HTML)) if format in ('all', 'text'): column.append( _linked_item(fp, outputdir, GitDiffSubcmd.REPORT_TEXT)) if options.pattern: if format in ('all', 'html'): column.append( _linked_item(fp, outputdir, GitDiffSubcmd.FILTER_HTML)) if format in ('all', 'text'): column.append( _linked_item(fp, outputdir, GitDiffSubcmd.FILTER_TEXT)) fp.section(' '.join([str(col) for col in column])) if order: with fp.table() as table: for sha1 in order: author, committer, subject = sha1s[sha1] if not remote: table.row(sha1, author, committer, subject) continue if options.gitiles: sha1a = fp.item(sha1[:20], '%s#q,%s' % (remote, sha1)) sha1b = fp.item( sha1[20:], '%s/plugins/gitiles/%s/+/%s^!' % (remote, node.name, sha1)) table.row(fp.item((sha1a, sha1b), tag='pre'), author, committer, subject) else: link = fp.item(sha1, '%s#q,%s' % (remote, sha1), tag='pre') table.row(link, author, committer, subject) return True
def execute(self, options, *args, **kws): # pylint: disable=R0915 SubCommand.execute(self, options, option_import=True, *args, **kws) logger = Logger.get_logger() # pylint: disable=E1101 pkgs, name = list(), None for pkg in args: pkgname, revision = _split_name( pkg, options.pkg_pattern, options.filter_out_chars) if name and pkgname != name: logger.warn( 'Warning: pkgname "%s" mismatched "%s"', pkgname, name) if not revision: logger.error( 'Error: %s failed to be recognized with revision' % pkg) else: name = pkgname pkgs.append((os.path.realpath(pkg), pkgname, revision)) if len(pkgs) != len(args): return if not options.keep_order: pkgs.sort(_pkg_sort) if options.show_order or options.verbose > 0: print 'Effective packages (%d)' % len(pkgs) print '----------------------------' for pkg, pkgname, revision in pkgs: print '%s %-15s %s' % (pkgname, '[v%s]' % revision, pkg) print if options.show_order: return RaiseExceptionIfOptionMissed( options.name, "project name (--name) is not set") RaiseExceptionIfOptionMissed( options.pkg_pattern, "pkg pattern (--pkg-pattern) is not set") RaiseExceptionIfOptionMissed( args, "no files or directories are specified to import") if not options.tryrun and options.remote: gerrit = Gerrit(options.remote) gerrit.create_project( options.name, description=options.description or False, options=options) branch = options.branch or 'master' path, _ = os.path.splitext(os.path.basename(options.name)) path = os.path.realpath(path) if options.offsite and not os.path.exists(path): os.makedirs(path) remote = '%s/%s' % (options.remote.rstrip(), options.name) \ if options.remote else None project = GitProject( options.name, worktree=path, gitdir='%s/.git' % path, revision=branch, remote=remote) ret = project.init_or_download( branch, single_branch=True, offsite=options.offsite) if ret != 0: logger.error('Failed to init the repo %s' % project) return False temp = options.temp_directory or tempfile.mkdtemp() tags = list() filter_out = list([r'\.git/']) for fout in options.filter_out or list(): filter_out.extend(fout.split(',')) for pkg, pkgname, revision in pkgs: workp = pkg tmpl = dict({ 'n': pkgname, 'name': pkgname, 'N': pkgname.upper(), 'NAME': pkgname.upper(), 'v': revision, 'version': revision, 'V': revision.upper(), 'VERSION': revision.upper()}) if options.message_template: message = options.message_template % tmpl else: message = 'Import %s' % ( '%s%s%s' % ( pkgname, revision and ' %s' % options.version_prefix, revision)) if options.use_commit_file: message = _read_file_with_escape( pkg, options.enable_escape, message) if os.path.isfile(pkg): FileUtils.extract_file(pkg, temp) workp = temp if options.init_path: inited = os.path.join(workp, options.init_path) if os.path.exists(inited): workp = inited if options.washed: if workp != temp: os.makedirs(temp) shutil.copytree(workp, temp, symlinks=True) # wash the directory washer = FileWasher( temp, overrideReadOnly=True, eol=options.washer_eol, tab=options.washer_tabsize > 0, trailing=options.washer_trailing) if washer.wash(temp): workp = temp if options.auto_detect: dname = os.listdir(workp) while 0 < len(dname) < 2: workp += '/%s' % dname[0] dname = os.listdir(workp) logger.info('Go into %s' % workp) if options.washed: diff = FileDiff(project.path, workp, filter_out, enable_sccs_pattern=options.filter_out_sccs) if diff.sync(project, logger) > 0: ret = 0 timestamp = diff.timestamp else: timestamp = _timestamp(workp) FileUtils.rmtree(project.path, ignore_list=(r'^\.git.*',)) FileUtils.copy_files(workp, project.path) ret = project.add('--all', project.path) if ret == 0: args = list() if options.author: args.append('--author="%s"' % options.author) args.append('-m') args.append(message) args.append('--date="%s"' % time.ctime(timestamp)) ret = project.commit(*args) if ret == 0: if options.version_template: tags.append(options.version_template % tmpl) elif options.local: trefs = self.override_value( # pylint: disable=E1101 options.refs, options.tag_refs) or '' if trefs: trefs += '/' tags.append( '%s%s%s' % ( trefs, options.version_prefix, revision)) else: tags.append( '%s%s' % (options.version_prefix, revision)) ret, _ = project.tag(tags[-1]) if os.path.lexists(temp): try: shutil.rmtree(temp) except OSError, e: logger.exception(e)
def execute(self, options, *args, **kws): # pylint: disable=R0915 SubCommand.execute(self, options, option_import=True, *args, **kws) logger = Logger.get_logger() # pylint: disable=E1101 ret, _, pkgs = PkgImportSubcmd.build_packages(options, args, logger) if not ret: return if options.show_order or (options.verbose and options.verbose > 0): print('Effective packages (%d)' % len(pkgs)) print('----------------------------') for pkg, pkgname, revision in pkgs: print('%s %-15s %s' % (pkgname, '[v%s]' % revision, pkg)) print if options.show_order: return RaiseExceptionIfOptionMissed( options.name, "project name (--name) is not set") RaiseExceptionIfOptionMissed( options.remote or options.offsite, 'remote (--remote) is set') RaiseExceptionIfOptionMissed( options.pkg_pattern or options.message_template, 'pkg pattern (--pkg-pattern) is not set') RaiseExceptionIfOptionMissed( args, "no files or directories are specified to import") if not options.dryrun and options.remote: gerrit = Gerrit(options.remote, options) gerrit.create_project( options.name, description=options.description or False, options=options) branch = options.branch or 'master' name, _ = os.path.splitext(os.path.basename(options.name)) path = os.path.join(options.working_dir, name) if options.offsite and not os.path.exists(path): os.makedirs(path) if options.remote: ulp = urlparse(options.remote) if not ulp.scheme: remote = 'git://%s/%s' % ( options.remote.strip('/'), options.name) else: remote = '%s/%s' % (options.remote.strip('/'), options.name) else: remote = '' project = GitProject( options.name, worktree=path, gitdir='%s/.git' % path, revision=branch, remote=remote) optgc = options.extra_values(options.extra_option, 'git-clone') ret = project.init_or_download( branch, single_branch=True, offsite=options.offsite, reference=optgc and optgc.reference) if ret != 0: logger.error('Failed to init the repo %s' % project) return False filters = list() if options.washed: filters = list([r'\.git/']) for fout in options.filter_out or list(): filters.extend(fout.split(',')) opti = Values.build( detect_root=options.auto_detect, directory=options.temp_directory, filter_sccs=options.filter_out_sccs, filters=filters, force=options.force, local=options.local, imports=True, prefix=options.version_prefix, refs=options.refs, tag_refs=options.tag_refs, tmpl_escape=options.enable_escape, tmpl_file=options.use_commit_file, tmpl_message=options.message_template, tmpl_version=options.version_template, washed=options.washed, extra=options.extra_values(options.extra_option, 'git-commit')) tags = list() for pkg, pkgname, revision in pkgs: workplace = pkg if options.init_path: inited = os.path.join(workplace, options.init_path) if os.path.exists(inited): workplace = inited _, ptags = PkgImportSubcmd.do_import( project, opti, pkgname, workplace, revision, logger=logger) if ptags: tags.extend(ptags) if not ret and not options.local: # pylint: disable=E1101 # push the branches if self.override_value( options.branches, options.all): ret = project.push_heads( branch, self.override_value( options.refs, options.head_refs), force=options.force, dryrun=options.dryrun) # push the tags if tags and self.override_value( options.tags, options.all): optp = Values.build(fullname=True) ret = project.push_tags( tags, self.override_value( options.refs, options.tag_refs), options=optp, force=options.force, dryrun=options.dryrun) # pylint: enable=E1101 return ret == 0
def execute(self, options, *args, **kws): SubCommand.execute(self, options, *args, **kws) RaiseExceptionIfOptionMissed(options.git or options.offsite, 'git url (--git-url) is not set') ulp = urlparse(options.name or '') RaiseExceptionIfOptionMissed( ulp.scheme or options.remote, 'Neither git name (--name) nor remote (--remote) is set') logger = self.get_logger() # pylint: disable=E1101 if ulp.scheme: remote = ulp.hostname projectname = ulp.path.strip('/') else: remote = options.remote projectname = self.get_name(options) # pylint: disable=E1101 remote = FileUtils.ensure_path(remote, prefix='git://', subdir=projectname, exists=False) working_dir = self.get_absolute_working_dir(options) # pylint: disable=E1101 project = GitProject( options.git, worktree=working_dir, gitdir=FileUtils.ensure_path( working_dir, subdir=None if options.bare else '.git'), revision=options.revision, remote=remote, bare=options.bare, pattern=GitCloneSubcmd.get_patterns(options) # pylint: disable=E1101 ) ret = 0 if not options.offsite: optgc = options.extra_values(options.extra_option, 'git-clone') ret = project.download(url=options.git, bare=options.bare, reference=optgc and optgc.reference) if ret != 0: raise DownloadError('%s: failed to fetch project' % project) ulp = urlparse(remote) # creat the project in the remote if ulp.scheme in ('ssh', 'git'): if not options.dryrun and options.remote and options.repo_create: gerrit = Gerrit(options.remote, options) gerrit.create_project(ulp.path.strip('/'), description=options.description, source=options.git, options=options) else: raise ProcessingError('%s: unknown scheme for remote "%s"' % (project, remote)) self.do_hook( # pylint: disable=E1101 'pre-push', options, dryrun=options.dryrun) optgp = options.extra_values(options.extra_option, 'git-push') # push the branches if self.override_value( # pylint: disable=E1101 options.all, options.heads): optp = Values.build(extra=optgp, push_all=options.all or options.revision is None, fullname=options.keep_name) res = project.push_heads( options.revision, self.override_value( # pylint: disable=E1101 options.refs, options.head_refs), options.head_pattern, options=optp, push_all=options.all or options.revision is None, fullname=options.keep_name, force=options.force, dryrun=options.dryrun) ret |= res if res: logger.error('Failed to push heads') # push the tags if self.override_value( # pylint: disable=E1101 options.all, options.tags): optp = Values.build(extra=optgp, fullname=options.keep_name) res = project.push_tags( None if options.all else options.tag, self.override_value( # pylint: disable=E1101 options.refs, options.tag_refs), options.tag_pattern, options=optp, force=options.force, dryrun=options.dryrun) ret |= res if res: logger.error('Failed to push tags') self.do_hook( # pylint: disable=E1101 'post-push', options, dryrun=options.dryrun) return ret