def main():
parser = OptionParser( usage = "usage: %prog --ref some_rel --chk other_rel")
parser.add_option("--ref",
dest = "ref",
help = "The description string of the reference release (eg: 12.0.X,rel_3,AtlasOffline)" )
parser.add_option("--chk",
dest = "chk",
help = "The description string of the to-be-compared release (eg: 12.0.X,rel_3 or 12.0.3)" )
options,args = parser.parse_args()
if options.ref is None or options.chk is None:
parser.print_help()
return 1
ref = options.ref
chk = options.chk
import PyCmt.Cmt as Cmt
diffs = Cmt.get_tag_diff(ref=ref, chk=chk, verbose=True)
if len(diffs) > 0:
return 1
return 0
def main():
parser = OptionParser(usage="usage: %prog --ref some_rel --chk other_rel")
parser.add_option(
"--ref",
dest="ref",
help=
"The description string of the reference release (eg: 12.0.X,rel_3,AtlasOffline)"
)
parser.add_option(
"--chk",
dest="chk",
help=
"The description string of the to-be-compared release (eg: 12.0.X,rel_3 or 12.0.3)"
)
options, args = parser.parse_args()
if options.ref is None or options.chk is None:
parser.print_help()
return 1
ref = options.ref
chk = options.chk
import PyCmt.Cmt as Cmt
diffs = Cmt.get_tag_diff(ref=ref, chk=chk, verbose=True)
if len(diffs) > 0:
return 1
return 0
def main(args):
"""Get the list of tag differences between 2 releases (CERN centric)
"""
import PyCmt.Cmt as Cmt
diffs = Cmt.get_tag_diff(ref=args.old, chk=args.new, verbose=True)
if len(diffs) > 0:
return 1
return 0
def mungtag(s, allow_use_cmt=False):
if allow_use_cmt:
try:
return mungtag(s)
except Exception:
import PyCmt.Cmt as Cmt
cmt = Cmt.CmtWrapper()
pkg = cmt.find_pkg(s.split('-')[0])
if pkg:
avn._packpath = os.path.join(pkg.path, pkg.name)
avn._path = None
if s in ('HEAD', 'TRUNK'):
thisurl = os.path.join(avn.root, avn.packpath, 'trunk')
elif s == 'THIS':
thisurl = avn.url
elif s.endswith('-branch'):
thisurl = os.path.join(avn.root, avn.packpath, 'branches', s)
else:
thisurl = os.path.join(avn.root, avn.packpath, 'tags', s)
if not (avn.path is None):
thisurl = os.path.join(thisurl, avn.path)
return thisurl
def __init__(self, pkg, env=None, shell=os, msg=msg):
self.pkg_infos = pkg.copy()
self.pkg_name = pkg['pkg_name']
self.pkg_ver = pkg['pkg_ver']
self.pkg_src = pkg['pkg_src']
self.pkg_install_dir = pkg['pkg_install_dir']
self.pkg_dest_dir = pkg['pkg_install_dir'] # not a typo
self.pkg_installarea_dir = pkg['pkg_installarea_dir']
self.pkg_build_dir = osp.join(pkg['pkg_root'], env['CMTCONFIG'],
'pkg-build-%s' % self.pkg_name)
import PyCmt.Cmt as Cmt
self._cmt = Cmt.CmtWrapper()
self.msg = msg
self.sh = shell
self._orig_dir = self.sh.getcwd()
self.env = env.copy() if isinstance(env, dict) else {}
for k, v in self.pkg_infos.iteritems():
self.env[k] = v
self.env['pkg_install_dir'] = self.pkg_install_dir
self.env['pkg_build_dir'] = self.pkg_build_dir
self.env['pkg_dest_dir'] = self.pkg_dest_dir
get = self.cmt
self.env['CMTINSTALLAREA'] = get(macro_value='CMTINSTALLAREA')
self.env['tag'] = get(macro_value='tag')
self.env.update(_cmt_to_autoconf(self.cmt))
for k in self.env:
if k.startswith('pkg_'):
self.msg.debug('expanding [%s] => [%s]', k, self[k])
setattr(self, k, self[k])
help = "The name of the project to inspect" )
sc = 0
report = []
(options, args) = parser.parse_args()
msg = _L.logging.getLogger("TabNanny")
msg.setLevel(_L.logging.INFO)
msg.info('TabNanny checker')
project_names = None
if len(args) > 0:
project_names = [ arg for arg in args if arg[0] != "-" ]
pass
cmt = PyCmt.CmtWrapper()
if options.project_name is None:
if project_names is None:
# take the top-root project name
options.project_name = cmt.projects_dag()[0].name
else:
# FIXME
# take the first one...
options.project_name = str(project_names[0])
# also allows to give a filename: early exit...
import os.path as osp
if osp.exists(options.project_name):
path = options.project_name
msg.info(':'*80)
msg.info('checking [%s]...', path)
options.output = 'proj_%s_cmt_pkg_shelve.db' % options.project.lower()
options.output = os.path.expanduser(options.output)
options.output = os.path.expandvars(options.output)
options.output = os.path.realpath(options.output)
options.log_lvl = options.log_lvl.upper()
if hasattr(L.logging, options.log_lvl):
lvl = getattr(L.logging, options.log_lvl)
else:
print "*" * 80
print "BootStrap: Unknown logging level [%s] !!" % options.log_lvl
print "BootStrap: Defaulting to [INFO]..."
print "*" * 80
lvl = L.logging.INFO
cmt = Cmt.CmtWrapper(lvl)
msg = cmt.msg
msg.info('building dependencies...')
## _allowed_output_db_formats = ('.pkl',)
## _ext = os.path.splitext (options.output)[1]
## if not _ext in _allowed_output_db_formats:
## msg.error ('unknown output-db format: [%s]', _ext)
## msg.error ('allowed formats are %r', _allowed_output_db_formats)
## raise SystemExit(1)
## if _ext == '.pkl':
## try:
## import cPickle as pickle
## except ImportError:
__version__ = "$Revision: 1.4 $" __author__ = "Frank Winklmeier" import sys import os import getopt import string import subprocess from PyCmt import Cmt try: import multiprocessing as mp except ImportError: mp = None cmt = Cmt.CmtWrapper() def usage(): print """\ Usage: pkgco.py [OPTION]... PACKAGE... Checkout PACKAGE from the release. Possible formats: Package find container and checkout tag in release Package-X-Y-Z find container and checkout specified tag Container/Package checkout tag in release Container/Package-X-Y-Z checkout specified tag where OPTION is: -A checkout the HEAD/trunk of the package(s) -f FILE contains PACKAGE list (one per line) -s only show package version, no checkout
if options.pkgName == None:
str(parser.print_help() or "")
sys.exit(1)
options.logLvl = options.logLvl.upper()
import PyCmt.Logging as L
if hasattr(L.logging, options.logLvl):
lvl = getattr(L.logging, options.logLvl)
else:
print "*" * 80
print "BootStrap: Unknown logging level [%s] !!" % options.logLvl
print "BootStrap: Defaulting to [INFO]..."
print "*" * 80
lvl = L.logging.INFO
cmt = Cmt.CmtWrapper(lvl)
cmt.msg.info("cmt show clients [%s]", options.pkgName)
clients = cmt.showClients(options.pkgName)
if options.doCheckOut:
def do_checkout(pkg):
cmt.checkOut(pkg.fullName(), pkg.version)
try:
import multiprocessing as mp
nworkers = min(mp.cpu_count(), 4)
workers = mp.Pool(processes=nworkers)
res = workers.map_async(func=do_checkout, iterable=clients)
res = res.get()
except ImportError: # no multiprocessing module
def get_tag_diffs(mon, ref, g, mon_task, ref_task):
client = AMIClient()
if not os.path.exists(AMI_CONFIG):
create_auth_config()
client.read_config(AMI_CONFIG)
mon_release = ''
ref_release = ''
mon_taginfo = get_configtags(client, mon.rtag)
ref_taginfo = get_configtags(client, ref.rtag)
configtags = [
'SWReleaseCache', 'lvl1ps', 'hltps', 'smk', 'enlvl1prescales'
]
configcomp = {}
for n, info in enumerate(mon_taginfo):
ref_info = ref_taginfo[n]
for xinfo in info.keys():
if xinfo in configtags:
if xinfo == 'SWReleaseCache': mon_rel = info[xinfo]
configcomp[xinfo] = [info[xinfo], ref_info[xinfo]]
for info in ref_taginfo:
for xinfo in info.keys():
if xinfo == 'SWReleaseCache': ref_rel = info[xinfo]
mon_release = mon_rel.replace('_', ',')
ref_release = ref_rel.replace('_', ',')
import PyCmt.Cmt as Cmt
diffs = Cmt.get_tag_diff(ref=ref_release, chk=mon_release, verbose=False)
g.write('<table>\n')
g.write('<tr><td width="250"></td><td width="250"><b>Reprocessing</b></td>'
'<td width="250"><b>Reference</b></tr>')
ami_link = '<a href ="https://ami.in2p3.fr/AMI/servlet/net.hep.atlas.Database.Bookkeeping.AMI.Servlet.Command?Converter=/AMIXmlToAMIProdHtml.xsl&Command=FormListConfigurationTag+-configTag=%s">%s</a></td>'
sav_link = '<a href="https://savannah.cern.ch/task/?%s"> Task #%s </a></td>'
g.write('<tr><td>AMI Tag </td>')
for tag in (mon.rtag, ref.rtag):
g.write(
'<td><a href ="https://ami.in2p3.fr/AMI/servlet/net.hep.atlas.Database.Bookkeeping.AMI.Servlet.Command?Converter=/AMIXmlToAMIProdHtml.xsl&Command=FormListConfigurationTag+-configTag=%s">%s</a></td>'
% (tag, tag))
g.write('</tr>')
g.write('<tr><td> Savannah Task </td>')
for task in (mon_task, ref_task):
if task == None:
g.write(
'<td><a href="https://savannah.cern.ch/task/index.php?go_report=Apply&group=atlas-trig&func=browse&category_id=107&status_id=0"> Search Tasks </a></td>'
)
else:
g.write(
'<td><a href="https://savannah.cern.ch/task/?%s"> Task #%s </a></td>'
% (task, task))
g.write('</tr>\n')
g.write('<tr><td> Run </td>')
for run in (mon.run, ref.run):
g.write('<td> %s </td>' % str(run))
g.write('</tr><tr></tr>\n')
g.write('<tr><td><b>Tag Configuration </b></td></tr>\n')
for field in configtags:
g.write('<tr><td>%s</td><td>%s</td><td>%s</td>' %
(field, configcomp[field][0], configcomp[field][1]))
g.write('</tr>\n')
g.write('<tr></tr>')
g.write('</table>')
g.write('<h3> Release Tag Differences </h3>')
g.write('<p> Found [%i] differences </p>\n' % len(diffs))
if len(diffs):
g.write('<table>\n')
g.write(
'<tr><td width = "150"><b>Reprocessing</b></td><td width="250"><b>mon-project</b></td>'
)
g.write(
'<td width = "150"><b>Reference</b></td><td width="250"><b>ref-project</b></td>'
)
g.write('<td width = "500"><b>package name</b></td></tr>')
for diff in diffs:
g.write('<tr>')
g.write('<td> %s </td>' % diff['chk'])
g.write('<td> %s </td>' % diff['chk_proj'])
g.write('<td> %s </td>' % diff['ref'])
g.write('<td> %s </td>' % diff['ref_proj'])
g.write('<td> %s </td>' % diff['full_name'])
g.write('</tr>\n')
g.write('</table>')
return 0
err = "*** problem running [cmt config] ! ***\n%s" % out
raise RuntimeError(err)
else:
msg.debug ('[cmt config] cmd was OK')
sc, out = commands.getstatusoutput ('%s show uses >| uses.%s.cmt'%\
(cmt.bin, options.project))
if sc != 0:
err = "*** problem running [cmt show uses] ! ***\n%s"%out
raise RuntimeError(err)
else:
msg.debug ('[cmt show uses >| uses.%s.cmt] cmd was OK',
options.project)
msg.info ('building package db...')
pkg_db = Cmt.buildPkgDb ('uses.%s.cmt'%options.project, msg)
msg.info ('building dependency graph...')
db = Cmt.buildDepGraph ('uses.%s.cmt'%options.project, pkg_db, msg)
os.chdir(cwd)
out_db[options.project] = db
out_db.close()
msg.info ('nbr of packages: %i', len(db.pkgs()))
## pkgs = db.pkgs().keys()[:10]
## for i in pkgs:
## msg.info('\t%s',i)
raise SystemExit(0)
def main(args):
"""submit one or more package tags to TagCollector
TAG can be one of the following formats:
Container/Package-00-01-02
Package-00-01-02
Package --> will use latest package tag
All submitted tags need approval via the TagCollector web interface.
If several TAGs are given they will be submitted to the same release(s)
with the same justification, etc. Optionally a bundle name can be specified.
If no release number is given a list of available releases is presented.
For any required argument that is not specified on the command line,
an interactive query is presented. Some text fields support history (arrow keys).
Authentication is handled via pyAMI (see https://atlas-ami.cern.ch/AMI/pyAMI)
"""
import PyUtils.AmiLib as amilib
client = amilib.Client()
def select_tag():
value = raw_input('Please select (q to quit): ')
if value.lower() == 'q':
raise StopIteration
return int(value)
# create a list of (pkg,tag) with full package path
pkgs = []
for pkg in args.pkgs:
# a file ?
if osp.exists(pkg):
fname = pkg
print "::: taking tags from file [%s]..." % (fname, )
for l in open(fname, 'r'):
l = l.strip()
if l:
print " - [%s]" % (l, )
pkgs.append(l)
else:
pkgs.append(pkg)
pkg_list = [
client.find_pkg(pkg, cbk_fct=select_tag, check_tag=False)
for pkg in pkgs
]
# setup history
readline.set_history_length(10)
# query release if project is known
if args.project and not args.release:
for p in args.project.split(','):
rel = client.get_open_releases(p)
if len(rel) == 0:
continue
if not args.release:
args.release = query_release(rel, p)
else:
args.release += (',%s' % query_release(rel, p))
if args.release and len(args.release.split(',')) == 1:
_release = args.release.split(',')[0]
args.release = ','.join([_release] * len(pkg_list))
# adjust the project list too
if args.project and len(args.project.split(',')) == 1:
args.project = ','.join([args.project.split(',')[0]] *
len(pkg_list))
# query project if release is known
if args.release and not args.project:
_releases = args.release.split(',')
_projects = []
rel = _releases[0]
for pkg in pkg_list:
proj = _get_projects(client, rel, pkg)
if len(proj) == 0:
_projects.append(None)
continue
v = query_project(proj, rel, pkg)
_projects.append(v)
pass # pkgs
if not args.project:
args.project = ','.join(_projects)
else:
args.project += ',' + ','.join(_projects)
pass
# Find latest tag if needed
print '-' * 80
for p in pkg_list:
if not 'packageTag' in p:
pkg = (p['packagePath'] + p['packageName']).strip(
'/') # CMTise path
p['packageTag'] = cmt.CmtWrapper().get_latest_pkg_tag(pkg)
print 'Using latest tag %s' % (p['packageTag'])
# query for missing options
for o in (
'project',
'release',
'justification',
'savannah',
):
value = getattr(args, o)
if value:
print '%s : %s' % (o, value)
else:
setattr(args, o, query_option(o))
print '-' * 80
args.project = args.project.split(',')
args.release = args.release.split(',')
if len(args.project) != len(args.release):
raise RuntimeError(
'Number of projects %s and releases %s do not match' %
(args.project, args.release))
# If only one tag given, submit this tag to all releases
if len(pkg_list) == 1: pkg_list = pkg_list * len(args.release)
choice = raw_input("Submit tag? [Y/n] ")
ok = len(choice) == 0 or choice.upper() == "Y"
if args.dry_run:
client.dry_run = args.dry_run
releases = args.release[:]
projects = args.project[:]
exitcode = 0
if ok:
# Submit tag request
for p, rel, proj in zip(pkg_list, releases, projects):
args.release = [rel]
args.project = [proj]
submit_tag(client, args, p['packagePath'] + p['packageName'],
p['packageTag'])
else:
print "Tag submission aborted"
exitcode = 1
return exitcode
def main(args):
"""Get the list of tag differences between 2 releases (CERN centric)
"""
print "===> Generating tag difference by calling CMT \n"
import PyCmt.Cmt as Cmt
diffs = Cmt.get_tag_diff(ref=args.old, chk=args.new, verbose=False)
if (args.package):
print "===> Will now dump the SVN diff for ", args.package
args.domain = None
args.files = None
else:
print "===> Will now dump the SVN diff for the packages"
if (args.domain):
print "Restricting to packages in the domain", args.domain
if (args.project):
print "Restricting to packages in the project", args.project
if (args.files):
print "... and will limit to file-level (i.e. ChangeLog) differences."
for line in diffs:
oldTag = line['ref']
oldProject = line['ref_proj']
newTag = line['chk']
newProject = line['chk_proj']
if (args.project and args.project != newProject):
continue
path = line['full_name']
domain = path.split('/')[0]
# print "domain=",domain
if (args.domain and args.domain != domain):
continue
package = path.split('/')[-1]
if (args.package and package != args.package):
continue
if ('N/A' in newProject):
print "Package", package, "removed from release. Skipping. \n"
continue
print
print '===> Getting SVN diff for package ' + package + ' at ' + path + '\n'
path = "svn+ssh://svn.cern.ch/reps/atlasoff/" + path
#path="$SVNOFF/"+path
if ('N/A' in oldProject):
print "New package, so showing diff from first version. \n"
oldTag = package + '-00-00-00' #TODO What if this isn't the first tag?
else:
oldTag = package + '-' + oldTag
newTag = package + '-' + newTag
# print path+'/tags/'+oldTag, path+'/tags/'+newTag
svnargs = [
"svn", "diff", path + '/tags/' + oldTag, path + '/tags/' + newTag
]
if (args.files):
svnargs.extend(["--depth", "files"])
try:
svndiff = check_call(svnargs)
if (svndiff):
for svnline in svndiff:
print svnline
except CalledProcessError:
print 'Some problem running SVN diff!'
if len(diffs) > 0:
return 1
return 0