def main(): parser = OptionParser( usage = "usage: %prog --ref some_rel --chk other_rel") parser.add_option("--ref", dest = "ref", help = "The description string of the reference release (eg: 12.0.X,rel_3,AtlasOffline)" ) parser.add_option("--chk", dest = "chk", help = "The description string of the to-be-compared release (eg: 12.0.X,rel_3 or 12.0.3)" ) options,args = parser.parse_args() if options.ref is None or options.chk is None: parser.print_help() return 1 ref = options.ref chk = options.chk import PyCmt.Cmt as Cmt diffs = Cmt.get_tag_diff(ref=ref, chk=chk, verbose=True) if len(diffs) > 0: return 1 return 0
def main(): parser = OptionParser(usage="usage: %prog --ref some_rel --chk other_rel") parser.add_option( "--ref", dest="ref", help= "The description string of the reference release (eg: 12.0.X,rel_3,AtlasOffline)" ) parser.add_option( "--chk", dest="chk", help= "The description string of the to-be-compared release (eg: 12.0.X,rel_3 or 12.0.3)" ) options, args = parser.parse_args() if options.ref is None or options.chk is None: parser.print_help() return 1 ref = options.ref chk = options.chk import PyCmt.Cmt as Cmt diffs = Cmt.get_tag_diff(ref=ref, chk=chk, verbose=True) if len(diffs) > 0: return 1 return 0
def main(args): """Get the list of tag differences between 2 releases (CERN centric) """ import PyCmt.Cmt as Cmt diffs = Cmt.get_tag_diff(ref=args.old, chk=args.new, verbose=True) if len(diffs) > 0: return 1 return 0
def mungtag(s, allow_use_cmt=False): if allow_use_cmt: try: return mungtag(s) except Exception: import PyCmt.Cmt as Cmt cmt = Cmt.CmtWrapper() pkg = cmt.find_pkg(s.split('-')[0]) if pkg: avn._packpath = os.path.join(pkg.path, pkg.name) avn._path = None if s in ('HEAD', 'TRUNK'): thisurl = os.path.join(avn.root, avn.packpath, 'trunk') elif s == 'THIS': thisurl = avn.url elif s.endswith('-branch'): thisurl = os.path.join(avn.root, avn.packpath, 'branches', s) else: thisurl = os.path.join(avn.root, avn.packpath, 'tags', s) if not (avn.path is None): thisurl = os.path.join(thisurl, avn.path) return thisurl
def __init__(self, pkg, env=None, shell=os, msg=msg): self.pkg_infos = pkg.copy() self.pkg_name = pkg['pkg_name'] self.pkg_ver = pkg['pkg_ver'] self.pkg_src = pkg['pkg_src'] self.pkg_install_dir = pkg['pkg_install_dir'] self.pkg_dest_dir = pkg['pkg_install_dir'] # not a typo self.pkg_installarea_dir = pkg['pkg_installarea_dir'] self.pkg_build_dir = osp.join(pkg['pkg_root'], env['CMTCONFIG'], 'pkg-build-%s' % self.pkg_name) import PyCmt.Cmt as Cmt self._cmt = Cmt.CmtWrapper() self.msg = msg self.sh = shell self._orig_dir = self.sh.getcwd() self.env = env.copy() if isinstance(env, dict) else {} for k, v in self.pkg_infos.iteritems(): self.env[k] = v self.env['pkg_install_dir'] = self.pkg_install_dir self.env['pkg_build_dir'] = self.pkg_build_dir self.env['pkg_dest_dir'] = self.pkg_dest_dir get = self.cmt self.env['CMTINSTALLAREA'] = get(macro_value='CMTINSTALLAREA') self.env['tag'] = get(macro_value='tag') self.env.update(_cmt_to_autoconf(self.cmt)) for k in self.env: if k.startswith('pkg_'): self.msg.debug('expanding [%s] => [%s]', k, self[k]) setattr(self, k, self[k])
help = "The name of the project to inspect" ) sc = 0 report = [] (options, args) = parser.parse_args() msg = _L.logging.getLogger("TabNanny") msg.setLevel(_L.logging.INFO) msg.info('TabNanny checker') project_names = None if len(args) > 0: project_names = [ arg for arg in args if arg[0] != "-" ] pass cmt = PyCmt.CmtWrapper() if options.project_name is None: if project_names is None: # take the top-root project name options.project_name = cmt.projects_dag()[0].name else: # FIXME # take the first one... options.project_name = str(project_names[0]) # also allows to give a filename: early exit... import os.path as osp if osp.exists(options.project_name): path = options.project_name msg.info(':'*80) msg.info('checking [%s]...', path)
options.output = 'proj_%s_cmt_pkg_shelve.db' % options.project.lower() options.output = os.path.expanduser(options.output) options.output = os.path.expandvars(options.output) options.output = os.path.realpath(options.output) options.log_lvl = options.log_lvl.upper() if hasattr(L.logging, options.log_lvl): lvl = getattr(L.logging, options.log_lvl) else: print "*" * 80 print "BootStrap: Unknown logging level [%s] !!" % options.log_lvl print "BootStrap: Defaulting to [INFO]..." print "*" * 80 lvl = L.logging.INFO cmt = Cmt.CmtWrapper(lvl) msg = cmt.msg msg.info('building dependencies...') ## _allowed_output_db_formats = ('.pkl',) ## _ext = os.path.splitext (options.output)[1] ## if not _ext in _allowed_output_db_formats: ## msg.error ('unknown output-db format: [%s]', _ext) ## msg.error ('allowed formats are %r', _allowed_output_db_formats) ## raise SystemExit(1) ## if _ext == '.pkl': ## try: ## import cPickle as pickle ## except ImportError:
__version__ = "$Revision: 1.4 $" __author__ = "Frank Winklmeier" import sys import os import getopt import string import subprocess from PyCmt import Cmt try: import multiprocessing as mp except ImportError: mp = None cmt = Cmt.CmtWrapper() def usage(): print """\ Usage: pkgco.py [OPTION]... PACKAGE... Checkout PACKAGE from the release. Possible formats: Package find container and checkout tag in release Package-X-Y-Z find container and checkout specified tag Container/Package checkout tag in release Container/Package-X-Y-Z checkout specified tag where OPTION is: -A checkout the HEAD/trunk of the package(s) -f FILE contains PACKAGE list (one per line) -s only show package version, no checkout
if options.pkgName == None: str(parser.print_help() or "") sys.exit(1) options.logLvl = options.logLvl.upper() import PyCmt.Logging as L if hasattr(L.logging, options.logLvl): lvl = getattr(L.logging, options.logLvl) else: print "*" * 80 print "BootStrap: Unknown logging level [%s] !!" % options.logLvl print "BootStrap: Defaulting to [INFO]..." print "*" * 80 lvl = L.logging.INFO cmt = Cmt.CmtWrapper(lvl) cmt.msg.info("cmt show clients [%s]", options.pkgName) clients = cmt.showClients(options.pkgName) if options.doCheckOut: def do_checkout(pkg): cmt.checkOut(pkg.fullName(), pkg.version) try: import multiprocessing as mp nworkers = min(mp.cpu_count(), 4) workers = mp.Pool(processes=nworkers) res = workers.map_async(func=do_checkout, iterable=clients) res = res.get() except ImportError: # no multiprocessing module
def get_tag_diffs(mon, ref, g, mon_task, ref_task): client = AMIClient() if not os.path.exists(AMI_CONFIG): create_auth_config() client.read_config(AMI_CONFIG) mon_release = '' ref_release = '' mon_taginfo = get_configtags(client, mon.rtag) ref_taginfo = get_configtags(client, ref.rtag) configtags = [ 'SWReleaseCache', 'lvl1ps', 'hltps', 'smk', 'enlvl1prescales' ] configcomp = {} for n, info in enumerate(mon_taginfo): ref_info = ref_taginfo[n] for xinfo in info.keys(): if xinfo in configtags: if xinfo == 'SWReleaseCache': mon_rel = info[xinfo] configcomp[xinfo] = [info[xinfo], ref_info[xinfo]] for info in ref_taginfo: for xinfo in info.keys(): if xinfo == 'SWReleaseCache': ref_rel = info[xinfo] mon_release = mon_rel.replace('_', ',') ref_release = ref_rel.replace('_', ',') import PyCmt.Cmt as Cmt diffs = Cmt.get_tag_diff(ref=ref_release, chk=mon_release, verbose=False) g.write('<table>\n') g.write('<tr><td width="250"></td><td width="250"><b>Reprocessing</b></td>' '<td width="250"><b>Reference</b></tr>') ami_link = '<a href ="https://ami.in2p3.fr/AMI/servlet/net.hep.atlas.Database.Bookkeeping.AMI.Servlet.Command?Converter=/AMIXmlToAMIProdHtml.xsl&Command=FormListConfigurationTag+-configTag=%s">%s</a></td>' sav_link = '<a href="https://savannah.cern.ch/task/?%s"> Task #%s </a></td>' g.write('<tr><td>AMI Tag </td>') for tag in (mon.rtag, ref.rtag): g.write( '<td><a href ="https://ami.in2p3.fr/AMI/servlet/net.hep.atlas.Database.Bookkeeping.AMI.Servlet.Command?Converter=/AMIXmlToAMIProdHtml.xsl&Command=FormListConfigurationTag+-configTag=%s">%s</a></td>' % (tag, tag)) g.write('</tr>') g.write('<tr><td> Savannah Task </td>') for task in (mon_task, ref_task): if task == None: g.write( '<td><a href="https://savannah.cern.ch/task/index.php?go_report=Apply&group=atlas-trig&func=browse&category_id=107&status_id=0"> Search Tasks </a></td>' ) else: g.write( '<td><a href="https://savannah.cern.ch/task/?%s"> Task #%s </a></td>' % (task, task)) g.write('</tr>\n') g.write('<tr><td> Run </td>') for run in (mon.run, ref.run): g.write('<td> %s </td>' % str(run)) g.write('</tr><tr></tr>\n') g.write('<tr><td><b>Tag Configuration </b></td></tr>\n') for field in configtags: g.write('<tr><td>%s</td><td>%s</td><td>%s</td>' % (field, configcomp[field][0], configcomp[field][1])) g.write('</tr>\n') g.write('<tr></tr>') g.write('</table>') g.write('<h3> Release Tag Differences </h3>') g.write('<p> Found [%i] differences </p>\n' % len(diffs)) if len(diffs): g.write('<table>\n') g.write( '<tr><td width = "150"><b>Reprocessing</b></td><td width="250"><b>mon-project</b></td>' ) g.write( '<td width = "150"><b>Reference</b></td><td width="250"><b>ref-project</b></td>' ) g.write('<td width = "500"><b>package name</b></td></tr>') for diff in diffs: g.write('<tr>') g.write('<td> %s </td>' % diff['chk']) g.write('<td> %s </td>' % diff['chk_proj']) g.write('<td> %s </td>' % diff['ref']) g.write('<td> %s </td>' % diff['ref_proj']) g.write('<td> %s </td>' % diff['full_name']) g.write('</tr>\n') g.write('</table>') return 0
err = "*** problem running [cmt config] ! ***\n%s" % out raise RuntimeError(err) else: msg.debug ('[cmt config] cmd was OK') sc, out = commands.getstatusoutput ('%s show uses >| uses.%s.cmt'%\ (cmt.bin, options.project)) if sc != 0: err = "*** problem running [cmt show uses] ! ***\n%s"%out raise RuntimeError(err) else: msg.debug ('[cmt show uses >| uses.%s.cmt] cmd was OK', options.project) msg.info ('building package db...') pkg_db = Cmt.buildPkgDb ('uses.%s.cmt'%options.project, msg) msg.info ('building dependency graph...') db = Cmt.buildDepGraph ('uses.%s.cmt'%options.project, pkg_db, msg) os.chdir(cwd) out_db[options.project] = db out_db.close() msg.info ('nbr of packages: %i', len(db.pkgs())) ## pkgs = db.pkgs().keys()[:10] ## for i in pkgs: ## msg.info('\t%s',i) raise SystemExit(0)
def main(args): """submit one or more package tags to TagCollector TAG can be one of the following formats: Container/Package-00-01-02 Package-00-01-02 Package --> will use latest package tag All submitted tags need approval via the TagCollector web interface. If several TAGs are given they will be submitted to the same release(s) with the same justification, etc. Optionally a bundle name can be specified. If no release number is given a list of available releases is presented. For any required argument that is not specified on the command line, an interactive query is presented. Some text fields support history (arrow keys). Authentication is handled via pyAMI (see https://atlas-ami.cern.ch/AMI/pyAMI) """ import PyUtils.AmiLib as amilib client = amilib.Client() def select_tag(): value = raw_input('Please select (q to quit): ') if value.lower() == 'q': raise StopIteration return int(value) # create a list of (pkg,tag) with full package path pkgs = [] for pkg in args.pkgs: # a file ? if osp.exists(pkg): fname = pkg print "::: taking tags from file [%s]..." % (fname, ) for l in open(fname, 'r'): l = l.strip() if l: print " - [%s]" % (l, ) pkgs.append(l) else: pkgs.append(pkg) pkg_list = [ client.find_pkg(pkg, cbk_fct=select_tag, check_tag=False) for pkg in pkgs ] # setup history readline.set_history_length(10) # query release if project is known if args.project and not args.release: for p in args.project.split(','): rel = client.get_open_releases(p) if len(rel) == 0: continue if not args.release: args.release = query_release(rel, p) else: args.release += (',%s' % query_release(rel, p)) if args.release and len(args.release.split(',')) == 1: _release = args.release.split(',')[0] args.release = ','.join([_release] * len(pkg_list)) # adjust the project list too if args.project and len(args.project.split(',')) == 1: args.project = ','.join([args.project.split(',')[0]] * len(pkg_list)) # query project if release is known if args.release and not args.project: _releases = args.release.split(',') _projects = [] rel = _releases[0] for pkg in pkg_list: proj = _get_projects(client, rel, pkg) if len(proj) == 0: _projects.append(None) continue v = query_project(proj, rel, pkg) _projects.append(v) pass # pkgs if not args.project: args.project = ','.join(_projects) else: args.project += ',' + ','.join(_projects) pass # Find latest tag if needed print '-' * 80 for p in pkg_list: if not 'packageTag' in p: pkg = (p['packagePath'] + p['packageName']).strip( '/') # CMTise path p['packageTag'] = cmt.CmtWrapper().get_latest_pkg_tag(pkg) print 'Using latest tag %s' % (p['packageTag']) # query for missing options for o in ( 'project', 'release', 'justification', 'savannah', ): value = getattr(args, o) if value: print '%s : %s' % (o, value) else: setattr(args, o, query_option(o)) print '-' * 80 args.project = args.project.split(',') args.release = args.release.split(',') if len(args.project) != len(args.release): raise RuntimeError( 'Number of projects %s and releases %s do not match' % (args.project, args.release)) # If only one tag given, submit this tag to all releases if len(pkg_list) == 1: pkg_list = pkg_list * len(args.release) choice = raw_input("Submit tag? [Y/n] ") ok = len(choice) == 0 or choice.upper() == "Y" if args.dry_run: client.dry_run = args.dry_run releases = args.release[:] projects = args.project[:] exitcode = 0 if ok: # Submit tag request for p, rel, proj in zip(pkg_list, releases, projects): args.release = [rel] args.project = [proj] submit_tag(client, args, p['packagePath'] + p['packageName'], p['packageTag']) else: print "Tag submission aborted" exitcode = 1 return exitcode
def main(args): """Get the list of tag differences between 2 releases (CERN centric) """ print "===> Generating tag difference by calling CMT \n" import PyCmt.Cmt as Cmt diffs = Cmt.get_tag_diff(ref=args.old, chk=args.new, verbose=False) if (args.package): print "===> Will now dump the SVN diff for ", args.package args.domain = None args.files = None else: print "===> Will now dump the SVN diff for the packages" if (args.domain): print "Restricting to packages in the domain", args.domain if (args.project): print "Restricting to packages in the project", args.project if (args.files): print "... and will limit to file-level (i.e. ChangeLog) differences." for line in diffs: oldTag = line['ref'] oldProject = line['ref_proj'] newTag = line['chk'] newProject = line['chk_proj'] if (args.project and args.project != newProject): continue path = line['full_name'] domain = path.split('/')[0] # print "domain=",domain if (args.domain and args.domain != domain): continue package = path.split('/')[-1] if (args.package and package != args.package): continue if ('N/A' in newProject): print "Package", package, "removed from release. Skipping. \n" continue print print '===> Getting SVN diff for package ' + package + ' at ' + path + '\n' path = "svn+ssh://svn.cern.ch/reps/atlasoff/" + path #path="$SVNOFF/"+path if ('N/A' in oldProject): print "New package, so showing diff from first version. \n" oldTag = package + '-00-00-00' #TODO What if this isn't the first tag? else: oldTag = package + '-' + oldTag newTag = package + '-' + newTag # print path+'/tags/'+oldTag, path+'/tags/'+newTag svnargs = [ "svn", "diff", path + '/tags/' + oldTag, path + '/tags/' + newTag ] if (args.files): svnargs.extend(["--depth", "files"]) try: svndiff = check_call(svnargs) if (svndiff): for svnline in svndiff: print svnline except CalledProcessError: print 'Some problem running SVN diff!' if len(diffs) > 0: return 1 return 0