def makeUSR(self):
logger.info( 'Making UserStuffRetriever Project ')
cfg = RTTConfiguration(self.rttConfFile)
confDict = cfg.config
#extend oringinal Config
confDict['originalBranch'] = cfg.originalBranch
confDict['findNightly'] = cfg.findNightly
confDict['valid'] = cfg.valid
confDict['isNightly'] = cfg.isNightly
if confDict['release'] == 'nightly':
lcr ='/afs/cern.ch/atlas/software/builds/nightlies/dev/AtlasOffline/latest_copied_release'
confDict['release'] = os.readlink(lcr)
print 'nightly Tag is '+str(os.readlink(lcr))
print formatCollection(confDict)
#print cfg.__str__()
paths = NoPackagePaths('RunTimeTester-00-00-96',
confDict, RunTimer(60*60*20 ,logger ),
'outLog.log', logger)
#usr = UserStuffRetrieverProject(paths,
#usr = UserStuffRetrieverRelease(paths,
# self.subString)
#usr = UserStuffRetrieverFactory(logger).create(paths, self.subString, confDict)
usr = UserStuffRetrieverFactory(logger).create(paths, self.subString)
return usr
class DataSetMaker:
def __init__(self, packageCfgFile, destDir, catalog=''):
self.packageCfg = packageCfgFile
self.catalog = catalog
self.destDir = destDir
def makeDatasets(self):
datasets = []
print '====================================='
print 'Config to parse ' + str(self.packageCfg)
print '====================================='
try:
dom = parse(self.packageCfg)
except Exception, e:
print '====================================='
print 'Error in parsing ' + str(self.packageCfg)
print '====================================='
print str(e)
return ([None], [None])
nonDQ2 = self.makeNonDQ2data(dom)
dq2 = self.makeDQ2data(dom)
return (nonDQ2, dq2)
print 'DQ2 data'
for d in dq2:
print formatCollection((d.__dict__))
print '-------------------------------------------------------------'
print 'Non DQ2 data'
for nd in nonDQ2:
print formatCollection(nd.__dict__)
def dumpLinkTable(table):
labelDict = {}
for key in table.keys():
childLabels = []
for child in table[key]:
childLabels.append(child.group)
if key == None:
print key, childLabels
else:
print key.group, childLabels
formatCollection(labelDict)
def __str__(self):
s = ''
for k in self.__dict__.keys():
if k != 'config':
s += '%s %s\n' % (k, str(self.__dict__[k]))
s += '----\nconfig: \n%s\n' % formatCollection(self.config)
return s
def __str__(self):
string ='group: %s\n'%self.group
string +='parent: %s\n'%self.parent
string +='log`Checker: %s\n'%str(self.logChecker)
string +='\n'
string += formatCollection(self.kitItems)
return string
class CopyDQ2Files:
def __init__(self , dsObject):
self.dataset = dsObject
self.name = self.dataset.logicalName
self.destDir = self.dataset.destDir
def filesString(self):
s = ''
for file in self.dataset.filesToCopy:
s += ' '+os.path.basename(file)
return s
def doCopy(self):
self.cpstatus = {}
stage,castor = self.setAreas()
print 'stage ' +stage
print 'castor ' +castor
files = [os.path.basename(file) for file in self.dataset.filesToCopy]
for file in files:
dq2Getcmd = 'dq2_get -s '+self.dataset.site
dq2Getcmd += ' -rc -d '+stage
dq2Getcmd += ' '+self.name
dq2Getcmd += ' '+file
temp = os.path.join(stage, file)
dest = os.path.join(castor, file)
try:
cmd = Popen4('rfdir '+dest)
result = str(cmd.fromchild.read())
if re.search('No such file', result):
print dq2Getcmd
fromGrid = Popen4(dq2Getcmd)
answer = fromGrid.fromchild.readlines()
self.handleStatus(self.name, answer, self.cpstatus)
#fromGrid.wait()
toCastor = Popen4('rfcp '+temp+ ' '+dest)
toCastor.wait()
else:
print 'File already available on castor'
print 'src file '+temp
print 'dest file '+dest
time.sleep(10)
rmtemp = Popen4('rm -f '+temp)
rmtemp.wait()
print 'deleted '+temp
except Exception, e:
print 'problem getting data from grid %s'%str(e)
if 'error' in self.cpstatus.values():
self.dataset.copyStatus = 'error'
self.dataset.copyInfo = str(formatCollection(self.cpstatus))
else:
self.dataset.copyStatus = 'success'
def makeUSR(self):
logger.info('Making UserStuffRetriever Monolithic')
confDict = rttConfiguration(self.rttConfFile)
logger.debug( 'rttConfig :'+formatCollection(str(confDict)))
paths = NoPackagePaths('RunTimeTester-00-00-89',
confDict, RunTimer(60*60*20 ,logger ),
'outLog.log')
usr = UserStuffRetriever(paths, self.subString, confDict )
return usr
def hasTop(self, nodes):
tops = [n for n in nodes if n.myLabel() == self.topLabel]
assert (len(tops) < 2)
if len(tops) == 0:
msg = 'No top node in Tree, raising exception\n'
msg += 'top node label %s \n' % self.topLabel
msg += formatCollection([n.myLabel() for n in nodes])
logger.error(msg)
raise RTTCodingError(msg)
top = tops[0]
top.parent = self.topParentLabel
def lineage(self, nodes):
for n in nodes:
# do not remove the top node
if n.myLabel() == self.topLabel: continue
aLabels = [a.myLabel() for a in self.ancestors(n)]
if self.topLabel not in aLabels:
msg = 'removing node %s ' % n.myLabel()
msg += 'which cannot be traced back to the top node'
msg += 'ancestors are:\n%s\n ' % (formatCollection(aLabels))
msg += 'top label is: %s' % self.topLabel
logger.error(msg)
nodes.remove(n)
def handleCmdOutput(self, output):
pattern = 'ERROR|not found'
for line in output:
print line
if re.search(pattern, line):
self.dataset.copyStatus = 'error'
self.dataset.copyInfo = line
elif re.search('Total:', line):
total, failed = line.split('-')
if string.strip(failed)[-1:] == '0':
if string.strip(total)[-1:] != '0':
self.dataset.copyStatus = 'success'
print 'sucess success,success, success,success,success,success'
self.dataset.copyInf = line
else:
self.dataset.copyInfo = formatCollection(output)
def getReply(self):
# remove the EOL if present
reply = [replace(line, '\n', '' ) for line in self.reply]
lenReply = len(reply)
if not self.silent and self.nLinesToReport>-1:
if lenReply < self.nLinesToReport:
m = 'Command: %s Reply:\n' % (str(self.cmd))
self.logger.debug(m)
[self.logger.debug('%s' % str(l)) for l in reply]
else:
msg = 'ShellCommand - '
msg += 'total number of lines in reply %d ' % lenReply
msg += 'Truncating to %d lines\n' % self.nLinesToReport
msg += formatCollection(reply[:self.nLinesToReport]
)
self.logger.debug(msg)
return reply
def __str__(self):
s = '----------- Minder ---------------\n'
s += ' done: ' + str(self.isDone()) + '\n'
s += ' weight: ' + str(self.jobWeight) + '\n'
s += ' runPath: ' + str(self.runPath) + '\n'
s += ' keepFilePatterns: ' + formatCollection(
self.keepFilePatterns) + '\n'
s += ' auxFilePatterns: ' + formatCollection(
self.auxFilePatterns) + '\n'
s += ' keepFiles: ' + formatCollection(
self.keepFiles) + '\n'
s += ' auxFiles: ' + formatCollection(self.auxFiles) + '\n'
s += ' actions: ' + formatCollection(self.actions) + '\n'
s += ' tests: ' + formatCollection(self.tests) + '\n'
s += ' scripts: ' + str(self.scripts.keys()) + '\n'
s += ' descriptor:\n'
return s
def makeUSR(self):
logger.info('Making UserStuffRetriever Project ')
cfg = RTTConfiguration(self.rttConfFile)
self.confDict = cfg.config
#extend original Config
self.confDict['originalBranch'] = cfg.originalBranch
self.confDict['findNightly'] = cfg.findNightly
self.confDict['valid'] = cfg.valid
self.confDict['isNightly'] = cfg.isNightly
self.RTTSrcDir = self.copierConfig.get('rttsrc')
self.RTTLibDir = self.copierConfig.get('rttLibDir')
self.logFileDir = self.copierConfig.get('logDir')
if self.confDict['release'] == 'nightly':
lcr = '/afs/cern.ch/atlas/software/builds/nightlies/'
lcr += self.confDict['originalBranch']
lcr += '/AtlasProduction/latest_copied_release'
self.confDict['release'] = os.readlink(lcr)
self.logger.info('nightly Tag is ' + str(os.readlink(lcr)))
self.logger.debug(formatCollection(self.confDict))
self.parametersForCMT = ParametersForCMT(self.confDict, self.logger)
self.paths = NoPackagePaths(self.RTTSrcDir, self.RTTLibDir,
self.confDict, self.logFileDir,
self.summaryDocument.createElement,
self.summaryDocument.createTextNode,
self.parametersForCMT, self.logger)
usr = UserStuffRetrieverFactory(logger).create(self.paths,
self.subString)
return usr
def __str__(self):
s = Minder.__str__(self)
s += '----------- Worker Minder ---------------\n'
s += ' checks: %s' % formatCollection(self.checks)
return s
def __str__(self):
str += self.__class__.__name__ + '\n'
str += formatCollection(self.__dict__) + '\n'
return str
def __str__(self):
str = 'TestDescriptor: module name %s test name %s \n' % (self.moduleName, self.testName)
str += 'argDict: \n%s' % formatCollection(self.argDict)
return str
def __str__(self):
str = 'GetUserConfigData - packages:\n'
str += formatCollection([p.name for p in self.packages])
return str
def __str__(self):
s = Minder.__str__(self)
names = [job.identifiedName for job in self.jobsToWatch]
s += 'job to watch.: %s' % formatCollection(names)
return s
def __str__(self):
return 'CVSPackage:\n'+formatCollection(self.__dict__)+'\n'