def insertApplicationToCS(self, name, csParameter):
"""add given application found via CVMFS to the CS"""
pars = dict(self.parameter)
pars['name'] = name
gLogger.notice("%(name)s: Adding version %(version)s to the CS" % pars)
existingVersions = gConfig.getSections(
"%(softSec)s/%(platform)s/%(name)s" % pars, [])
if not existingVersions['OK']:
gLogger.error("Could not find all versions available in CS: %s" %
existingVersions['Message'])
dexit(255)
if pars['version'] in existingVersions['Value']:
gLogger.always(
'Application %s %s for %s already in CS, nothing to do' %
(name.lower(), pars['version'], pars['platform']))
return S_OK()
csPath = self.softSec + ("/%(platform)s/%(name)s/%(version)s/" % pars)
for par, val in csParameter.iteritems():
gLogger.notice("Add: %s = %s" % (csPath + par, val))
result = self.csAPI.setOption(csPath + par, val)
if result['OK']:
self.modifiedCS = True
else:
gLogger.error("Failure to add to CS", result['Message'])
return S_ERROR("")
return S_OK()
def __getOptionsFromCS(self,
path="/Website/Launchpad/Options",
delimiter=","):
gLogger.info("start __getOptionsFromCS")
result = gConfig.getOptionsDict(path)
gLogger.always(result)
if not result["OK"]:
return []
options = result["Value"]
for i in options.keys():
options[i] = options[i].split(delimiter)
result = gConfig.getSections(path)
if result["OK"]:
sections = result["Value"]
if len(sections) > 0:
for i in sections:
options[i] = self.__getOptionsFromCS(path + '/' + i, delimiter)
gLogger.always("options: %s" % options)
gLogger.info("end __getOptionsFromCS")
return options
def printOperation( indexOperation, verbose = True, onlyFailed = False ):
global output
i, op = indexOperation
prStr = ''
if op.SourceSE:
prStr += 'SourceSE: %s' % op.SourceSE
if op.TargetSE:
prStr += ( ' - ' if prStr else '' ) + 'TargetSE: %s' % op.TargetSE
if prStr:
prStr += ' - '
prStr += 'Created %s, Updated %s' % ( op.CreationTime, op.LastUpdate )
if op.Type == 'ForwardDISET':
from DIRAC.Core.Utilities import DEncode
decode, _length = DEncode.decode( op.Arguments )
if verbose:
output = ''
prettyPrint( decode, offset = 10 )
prStr += '\n Arguments:\n' + output.strip( '\n' )
else:
prStr += '\n Service: %s' % decode[0][0]
gLogger.always( " [%s] Operation Type='%s' ID=%s Order=%s Status='%s'%s%s" % ( i, op.Type, op.OperationID,
op.Order, op.Status,
( " Error='%s'" % op.Error ) if op.Error and op.Error.strip() else "",
( " Catalog=%s" % op.Catalog ) if op.Catalog else "" ) )
if prStr:
gLogger.always( " %s" % prStr )
for indexFile in enumerate( op ):
if not onlyFailed or indexFile[1].Status == 'Failed':
printFile( indexFile )
def __request( self ):
gLogger.info( "!!! PARAMS: ", str( self.request.arguments ) )
req = {}
start = 0
limit = 25
if self.request.arguments.has_key( "limit" ) and len( self.request.arguments["limit"][0] ) > 0:
limit = int( self.request.arguments["limit"][0] )
if self.request.arguments.has_key( "start" ) and len( self.request.arguments["start"][0] ) > 0:
start = int( self.request.arguments["start"][0] )
try:
sortDirection = str( self.request.arguments[ 'sortDirection' ] ).strip()
except:
sortDirection = "ASC"
try:
sortField = str( self.request.arguments[ 'sortField' ] ).strip()
except:
sortField = "UserName"
sort = [[sortField, sortDirection]]
gLogger.info( "!!! S O R T : ", sort )
if "username" in self.request.arguments:
users = list( json.loads( self.request.arguments[ 'username' ][-1] ) )
if len( users ) > 0:
req['UserName'] = users
if "usergroup" in self.request.arguments:
usersgroup = list( json.loads( self.request.arguments[ 'usergroup' ][-1] ) )
if len( usersgroup ) > 0:
req['UserGroup'] = usersgroup
if "usersgroup" in self.request.arguments and len( self.request.arguments["persistent"] ) > 0:
if str( self.request.arguments["persistent"] ) in ["True", "False"]:
req["PersistentFlag"] = str( self.request.arguments["persistent"] )
before = False
after = False
if self.request.arguments.has_key( "expiredBefore" ) and len( self.request.arguments["expiredBefore"] ) > 0:
try:
before = int( self.request.arguments["expiredBefore"] )
except:
pass
if self.request.arguments.has_key( "expiredAfter" ) and len( self.request.arguments["expiredAfter"] ) > 0:
try:
after = int( self.request.arguments["expiredAfter"] )
except:
pass
if before and after:
if before > after:
req["beforeDate"] = before
req["afterDate"] = after
else:
if before:
req["beforeDate"] = before
if after:
req["afterDate"] = after
gLogger.always( "REQUEST:", req )
return ( start, limit, sort, req )
def web_getMetadataFilesInFile(self):
self.set_header('Content-type', 'text/plain')
self.set_header('Content-Disposition',
'attachment; filename="error.txt"')
req = self.__request_file()
gLogger.always(req)
gLogger.debug("submit: incoming request %s" % req)
result = yield self.threadTask(self.fc.findFilesByMetadata,
req["selection"], req["path"])
if not result["OK"]:
gLogger.error("submit: %s" % result["Message"])
self.finish({"success": "false", "error": result["Message"]})
return
result = result["Value"]
retStrLines = []
if len(result) > 0:
#for key , value in result.items() :
for key, value in result.items():
for fileName in value:
retStrLines.append(key + "/" + fileName)
strData = "\n".join(retStrLines)
self.set_header('Content-type', 'text/plain')
self.set_header(
'Content-Disposition',
'attachment; filename="%s.txt"' % md5(str(req)).hexdigest())
self.set_header('Content-Length', len(strData))
self.finish(strData)
def __requestParams(self):
'''
We receive the request and we parse it, in this case, we are doing nothing,
but it can be certainly more complex.
'''
gLogger.always("!!! PARAMS: ", str(self.request.arguments))
responseParams = {
'element': None,
'name': None,
'elementType': None,
'statusType': None,
'status': None,
'tokenOwner': None,
'lastCheckTime': None,
'action': None
}
for key in responseParams:
if key in self.request.arguments and str(
self.request.arguments[key][-1]):
responseParams[key] = list(
json.loads(self.request.arguments[key][-1]))
return responseParams
def test_DaVinci_2_Production_PR33857_7(self):
""" Using ProdConf (production style). systemConfig = 'x86_64-slc6-gcc49-opt'
This is taken from PR 33857
"""
gLogger.always("**** DAVINCI v41r3")
ra = RunApplication()
ra.applicationName = 'DaVinci'
ra.applicationVersion = 'v41r3'
ra.systemConfig = 'x86_64-slc6-gcc49-opt'
ra.commandOptions = ['$APPCONFIGOPTS/Merging/DVMergeDST.py',
'$APPCONFIGOPTS/DaVinci/DataType-2012.py',
'$APPCONFIGOPTS/Merging/WriteFSR.py',
'$APPCONFIGOPTS/Merging/MergeFSR.py']
ra.extraPackages = [('AppConfig', 'v3r305'),
('ProdConf', '')
]
ra.step_Number = 1
ra.prodConfFileName = find_all('test_prodConf_davinci_v41r3.py', '..')[0]
ra.applicationLog = '00033857_00000007_7_log.txt'
ra.stdError = '00033857_00000007_7_err.txt'
res = ra.run()
self.assertTrue(res['OK'])
self.assertEqual(res['Value'], (0, '', ''))
def test_Brunel_Production_PR33857_5(self):
""" Using ProdConf (production style). systemConfig = 'x86_64-slc5-gcc46-opt'
This is taken from PR 33857
"""
gLogger.always("**** BRUNEL v43r2p11")
ra = RunApplication()
ra.applicationName = 'Brunel'
ra.applicationVersion = 'v43r2p11'
ra.systemConfig = 'x86_64-slc5-gcc46-opt'
ra.commandOptions = ['$APPCONFIGOPTS/Brunel/DataType-2012.py',
'$APPCONFIGOPTS/Brunel/MC-WithTruth.py',
'$APPCONFIGOPTS/Brunel/Sim09-Run1.py',
'$APPCONFIGOPTS/Persistency/DST-multipleTCK-2012.py',
'$APPCONFIGOPTS/Persistency/Compression-ZLIB-1.py']
ra.extraPackages = [('AppConfig', 'v3r302'),
('ProdConf', '')
]
ra.prodConfFileName = find_all('test_prodConf_brunel_v43r2p11.py', '..')[0]
ra.applicationLog = '00033857_00000005_5_log.txt'
ra.stdError = '00033857_00000005_5_err.txt'
res = ra.run()
self.assertTrue(res['OK'])
self.assertEqual(res['Value'], (0, '', ''))
def test_DaVinci_1_Production_PR33857_6(self):
""" Using ProdConf (production style). systemConfig = 'x86_64-slc5-gcc46-opt'
This is taken from PR 33857
"""
gLogger.always("**** DAVINCI v32r2p1")
ra = RunApplication()
ra.applicationName = 'DaVinci'
ra.applicationVersion = 'v32r2p1'
ra.systemConfig = 'x86_64-slc5-gcc46-opt'
ra.commandOptions = ['$CHARMCONFIGOPTS/MCFiltering/D02K3PiFromB2DstmunuXStripTrigFiltering_2012.py',
'$APPCONFIGOPTS/DaVinci/DataType-2012.py',
'$APPCONFIGOPTS/DaVinci/InputType-DST.py',
'$APPCONFIGOPTS/Persistency/Compression-ZLIB-1.py']
ra.extraPackages = [('AppConfig', 'v3r305'),
('CharmConfig', 'v3r30'),
('ProdConf', '')
]
ra.step_Number = 1
ra.prodConfFileName = find_all('test_prodConf_davinci_v32r2p1.py', '..')[0]
ra.applicationLog = '00033857_00000006_6_log.txt'
ra.stdError = '00033857_00000006_6_err.txt'
res = ra.run()
self.assertTrue(res['OK'])
self.assertEqual(res['Value'], (0, '', ''))
def __aftermath( self ):
action = self.action
success = ", ".join( self.actionSuccess )
failure = "\n".join( self.actionFailed )
if len( self.actionSuccess ) > 1:
sText = self.prefix + "s"
else:
sText = self.prefix
if len( self.actionFailed ) > 1:
fText = self.prefix + "s"
else:
fText = self.prefix
if len( success ) > 0 and len( failure ) > 0:
sMessage = "%s %sed successfully: " % ( sText , action , success)
fMessage = "Failed to %s %s:\n%s" % ( action , fText , failure )
result = sMessage + "\n\n" + fMessage
return { "success" : "true" , "result" : result }
elif len( success ) > 0 and len( failure ) < 1:
result = "%s %sed successfully: %s" % ( sText , action , success )
return { "success" : "true" , "result" : result }
elif len( success ) < 1 and len( failure ) > 0:
result = "Failed to %s %s:\n%s" % ( action , fText , failure )
gLogger.always( result )
return { "success" : "false" , "error" : result }
else:
result = "No action has performed due technical failure. Check the logs please"
gLogger.debug( result )
return { "success" : "false" , "error" : result }
def test_Boole_2_Production_PR33857(self):
""" Using ProdConf (production style). Taken from PR 33857.
Same as before but using "ANY" as CMT config
"""
gLogger.always("**** Boole v30r1")
ra = RunApplication()
ra.applicationName = 'Boole'
ra.applicationVersion = 'v30r1'
ra.commandOptions = ['$APPCONFIGOPTS/Boole/Default.py',
'$APPCONFIGOPTS/Boole/DataType-2012.py',
'$APPCONFIGOPTS/Boole/NoPacking.py',
'$APPCONFIGOPTS/Boole/Boole-SetOdinRndTrigger.py',
'$APPCONFIGOPTS/Persistency/Compression-ZLIB-1.py']
ra.extraPackages = [('AppConfig', 'v3r266'),
('ProdConf', '')
]
ra.step_Number = 1
ra.prodConfFileName = find_all('test_prodConf_boole_v30r1_2.py', '..')[0]
ra.applicationLog = '00033857_00000002_3_log.txt'
ra.stdError = '00033857_00000002_3_err.txt'
res = ra.run()
self.assertTrue(res['OK'])
self.assertEqual(res['Value'], (0, '', ''))
def sweeper( cls ):
""" move cached request to the central request manager
:param self: self reference
"""
cacheDir = cls.cacheDir()
# # cache dir empty?
if not os.listdir( cacheDir ):
gLogger.always( "sweeper: CacheDir %s is empty, nothing to do" % cacheDir )
return S_OK()
else:
# # read 10 cache dir files, the oldest first
cachedRequests = [ os.path.abspath( requestFile ) for requestFile in
sorted( filter( os.path.isfile,
[ os.path.join( cacheDir, requestName )
for requestName in os.listdir( cacheDir ) ] ),
key = os.path.getctime ) ][:10]
# # set cached requests to the central RequestManager
for cachedFile in cachedRequests:
# # break if something went wrong last time
try:
requestJSON = "".join( open( cachedFile, "r" ).readlines() )
cachedRequest = json.loads( requestJSON )
cachedName = cachedRequest.get( "RequestName", "***UNKNOWN***" )
putRequest = cls.requestManager().putRequest( requestJSON )
if not putRequest["OK"]:
gLogger.error( "sweeper: unable to set request %s @ ReqManager: %s" % ( cachedName,
putRequest["Message"] ) )
continue
gLogger.info( "sweeper: successfully put request '%s' @ ReqManager" % cachedName )
os.unlink( cachedFile )
except Exception, error:
gLogger.exception( "sweeper: hit by exception %s" % str( error ) )
return S_ERROR( "sweeper: hit by exception: %s" % str( error ) )
return S_OK()
def printFile(indexFile):
j, f = indexFile
gLogger.always(
" [%02d] ID=%s LFN='%s' Status='%s'%s%s" %
(j + 1, f.FileID, f.LFN, f.Status,
(" Error='%s'" % f.Error) if f.Error and f.Error.strip() else "",
(" Attempts=%d" % f.Attempt) if f.Attempt > 1 else ""))
def __fileType(self, fileType=None, returnList=False):
"""
return the file types taking into account the expected file types
"""
gLogger.verbose("BKQuery.__fileType: %s, fileType: %s" %
(self, fileType))
if not fileType:
return []
self.__getAllBKFileTypes()
if isinstance(fileType, list):
fileTypes = fileType
else:
fileTypes = fileType.split(',')
allRequested = None
if fileTypes[0].lower() == "all":
allRequested = True
bkTypes = self.getBKFileTypes()
gLogger.verbose('BKQuery.__fileType: bkTypes %s' % str(bkTypes))
if bkTypes:
fileTypes = list(set(bkTypes) - self.__exceptFileTypes)
else:
fileTypes = []
expandedTypes = set()
# print "Requested", fileTypes
for fileType in fileTypes:
if fileType.lower() == 'all.hist':
allRequested = False
expandedTypes.update([
t for t in self.__exceptFileTypes.union(self.__bkFileTypes)
if t.endswith('HIST')
])
elif fileType.lower().find("all.") == 0:
ext = '.' + fileType.split('.')[1]
fileType = []
if allRequested is None:
allRequested = True
expandedTypes.update([
t for t in set(self.getBKFileTypes()) -
self.__exceptFileTypes if t.endswith(ext)
])
else:
expandedTypes.add(fileType)
# Remove __exceptFileTypes only if not explicitly required
# print "Obtained", fileTypes, expandedTypes
gLogger.verbose(
"BKQuery.__fileType: requested %s, expanded %s, except %s" %
(allRequested, expandedTypes, self.__exceptFileTypes))
if expandedTypes - self.__bkFileTypes and not self.__alreadyWarned:
self.__alreadyWarned = True
gLogger.always(
"**** Take care: some requested file types do not exist!!",
str(sorted(expandedTypes - self.__bkFileTypes)))
if allRequested or not expandedTypes & self.__exceptFileTypes:
expandedTypes -= self.__exceptFileTypes
gLogger.verbose("BKQuery.__fileType: result %s" %
sorted(expandedTypes))
if len(expandedTypes) == 1 and not returnList:
return list(expandedTypes)[0]
else:
return list(expandedTypes)
def submit(self, monitor=False, printOutput=True):
""" submit FTS job
:param self: self reference
:param bool monitor: flag to monitor progress of FTS job
:param bool printOutput: flag to print output of execution to stdout
"""
res = self.__prepareForSubmission()
if not res['OK']:
return res
res = self.__submitFTSTransfer()
if not res['OK']:
return res
resDict = {
'ftsGUID': self.ftsGUID,
'ftsServer': self.ftsServer,
'submittedFiles': self.submittedFiles
}
if monitor or printOutput:
gLogger.always("Submitted %s@%s" % (self.ftsGUID, self.ftsServer))
if monitor:
self.monitor(untilTerminal=True,
printOutput=printOutput,
full=False)
return S_OK(resDict)
def __getDirectories(self):
""" get the directories where to look into (they are either given, or taken from the transformation ID
"""
if self.directories:
directories = []
printout = False
for directory in self.directories:
if not directory.endswith('...'):
directories.append(directory)
else:
printout = True
topDir = os.path.dirname(directory)
res = self.fileCatalog.listDirectory(topDir)
if not res['OK']:
# DError(errno.ENOENT, res['Message'] )
return S_ERROR(errno.ENOENT, res['Message'])
else:
matchDir = directory.split('...')[0]
directories += [d for d in res['Value']['Successful'].get(topDir, {}).get('SubDirs', [])
if d.startswith(matchDir)]
if printout:
gLogger.always('Expanded list of %d directories:\n%s' %
(len(directories), '\n'.join(directories)))
return directories
else:
return S_ERROR(errno.ENOENT, 'Need to specify the directories')
def run(self):
""" The main watchdog execution method
"""
result = self.initialize()
if not result['OK']:
gLogger.always('Can not start watchdog for the following reason')
gLogger.always(result['Message'])
return result
try:
while True:
gLogger.debug('Starting watchdog loop # %d' % self.count)
start_cycle_time = time.time()
result = self.execute()
exec_cycle_time = time.time() - start_cycle_time
if not result['OK']:
gLogger.error("Watchdog error during execution",
result['Message'])
break
elif result['Value'] == "Ended":
break
self.count += 1
if exec_cycle_time < self.pollingTime:
time.sleep(self.pollingTime - exec_cycle_time)
return S_OK()
except Exception:
gLogger.exception()
return S_ERROR('Exception')
def requestOptimization( self, job ):
jid = job.getJobId()
gLogger.info( "Sending job %d to optimizer %s" % ( jid, self.__name ) )
self.__lock.acquire()
try:
if self.__frozen:
return S_ERROR( "Optimizer is frozen" )
if jid in self.__jobsInOptimizer:
gLogger.warn( "Job %d already in optimizer %s" % ( jid, self.__name ) )
return S_OK()
if jid in self.__bookings:
del( self.__bookings[ jid ] )
self.__jobsInOptimizer.add( jid )
finally:
self.__lock.release()
result = job.dumpToStub()
if not result[ 'OK' ]:
return result
jobStub = result[ 'Value' ]
gLogger.always( "Sending job %s to %s [%s]" % ( jid, self.__name, self.__trid ) )
result = self.__msgSender.sendMessage( self.__trid, 'optimizeJob', jid, jobStub )
if not result[ 'OK' ]:
self.jobExitedOptimizer( jid )
return result
def __getXRSLExtraString(self, multiprocessor=False):
# For the XRSL additional string from configuration - only done at initialisation time
# If this string changes, the corresponding (ARC) site directors have to be restarted
#
# Variable = XRSLExtraString (or for multi processor mode)
# Default value = ''
# If you give a value, I think it should be of the form
# (aaa = "xxx")
# Otherwise the ARC job description parser will have a fit
# Locations searched in order :
# Top priority : Resources/Sites/<Grid>/<Site>/CEs/<CE>/XRSLExtraString
# Second priority : Resources/Sites/<Grid>/<Site>/XRSLExtraString
# Default : Resources/Computing/CEDefaults/XRSLExtraString
#
xrslExtraString = '' # Start with the default value
result = getCESiteMapping(self.ceHost)
if not result['OK'] or not result['Value']:
gLogger.error("Unknown CE ...")
return
self.site = result['Value'][self.ceHost]
# Now we know the site. Get the grid
grid = self.site.split(".")[0]
# The different possibilities that we have agreed upon
if multiprocessor:
xtraVariable = "XRSLMPExtraString"
else:
xtraVariable = "XRSLExtraString"
firstOption = "Resources/Sites/%s/%s/CEs/%s/%s" % (
grid, self.site, self.ceHost, xtraVariable)
secondOption = "Resources/Sites/%s/%s/%s" % (grid, self.site,
xtraVariable)
defaultOption = "Resources/Computing/CEDefaults/%s" % xtraVariable
# Now go about getting the string in the agreed order
gLogger.debug("Trying to get %s : first option %s" %
(xtraVariable, firstOption))
result = gConfig.getValue(firstOption, defaultValue='')
if result != '':
xrslExtraString = result
gLogger.debug("Found %s : %s" % (xtraVariable, xrslExtraString))
else:
gLogger.debug("Trying to get %s : second option %s" %
(xtraVariable, secondOption))
result = gConfig.getValue(secondOption, defaultValue='')
if result != '':
xrslExtraString = result
gLogger.debug("Found %s : %s" %
(xtraVariable, xrslExtraString))
else:
gLogger.debug("Trying to get %s : default option %s" %
(xtraVariable, defaultOption))
result = gConfig.getValue(defaultOption, defaultValue='')
if result != '':
xrslExtraString = result
gLogger.debug("Found %s : %s" %
(xtraVariable, xrslExtraString))
if xrslExtraString:
gLogger.always("%s : %s" % (xtraVariable, xrslExtraString))
gLogger.always(" --- to be added to pilots going to CE : %s" %
self.ceHost)
return xrslExtraString
def printRequest( request, status = None, full = False, verbose = True, terse = False ):
from DIRAC.DataManagementSystem.Client.FTSClient import FTSClient
global output
ftsClient = FTSClient()
if full:
output = ''
prettyPrint( request.toJSON()['Value'] )
gLogger.always( output )
else:
if not status:
status = request.Status
gLogger.always( "Request name='%s' ID=%s Status='%s'%s%s%s" % ( request.RequestName,
request.RequestID,
request.Status, " ('%s' in DB)" % status if status != request.Status else '',
( " Error='%s'" % request.Error ) if request.Error and request.Error.strip() else "" ,
( " Job=%s" % request.JobID ) if request.JobID else "" ) )
gLogger.always( "Created %s, Updated %s" % ( request.CreationTime, request.LastUpdate ) )
if request.OwnerDN:
gLogger.always( "Owner: '%s', Group: %s" % ( request.OwnerDN, request.OwnerGroup ) )
for indexOperation in enumerate( request ):
op = indexOperation[1]
if not terse or op.Status == 'Failed':
printOperation( indexOperation, verbose, onlyFailed = terse )
# Check if FTS job exists
res = ftsClient.getFTSJobsForRequest( request.RequestID )
if res['OK']:
ftsJobs = res['Value']
if ftsJobs:
gLogger.always( ' FTS jobs associated: %s' % ','.join( ['%s (%s)' % ( job.FTSGUID, job.Status ) \
for job in ftsJobs] ) )
def submit(self):
pagestart = time()
RPC = getRPCClient("WorkloadManagement/JobMonitoring")
user = str(credentials.getUsername())
result = RPC.getOwners()
if result["OK"]:
defaultGroup = gConfig.getValue("/Registry/DefaultGroup","")
if defaultGroup == "":
return {"success":"false","error":"Option /Registry/DefaultGroup is undefined, please set the default group in the CS"}
group = str(credentials.getSelectedGroup())
groupProperty = credentials.getProperties(group)
if user not in result["Value"] and ( "JobAdministrator" or "JobSharing" ) not in groupProperty:
c.result = {"success":"false","error":"You don't have any jobs in the DIRAC system"}
return c.result
else:
c.result = {"success":"false","error":result["Message"]}
return c.result
req = self.__request()
gLogger.always("getJobPageSummaryWeb(%s,%s,%s,%s)" % (req,globalSort,pageNumber,numberOfJobs))
result = RPC.getJobPageSummaryWeb(req,globalSort,pageNumber,numberOfJobs)
gLogger.always(" - REZ: " %result)
if result["OK"]:
result = result["Value"]
gLogger.info("ReS",result)
if result.has_key("TotalRecords"):
if result["TotalRecords"] > 0:
if result.has_key("ParameterNames") and result.has_key("Records"):
if len(result["ParameterNames"]) > 0:
if len(result["Records"]) > 0:
c.result = []
jobs = result["Records"]
head = result["ParameterNames"]
headLength = len(head)
for i in jobs:
tmp = {}
for j in range(0,headLength):
tmp[head[j]] = i[j]
c.result.append(tmp)
total = result["TotalRecords"]
timestamp = Time.dateTime().strftime("%Y-%m-%d %H:%M [UTC]")
if result.has_key("Extras"):
st = self.__dict2string(req)
extra = result["Extras"]
c.result = {"success":"true","result":c.result,"total":total,"extra":extra,"request":st,"date":timestamp}
else:
c.result = {"success":"true","result":c.result,"total":total,"date":timestamp}
else:
c.result = {"success":"false","result":"","error":"There are no data to display"}
else:
c.result = {"success":"false","result":"","error":"ParameterNames field is missing"}
else:
c.result = {"success":"false","result":"","error":"Data structure is corrupted"}
else:
c.result = {"success":"false","result":"","error":"There were no data matching your selection"}
else:
c.result = {"success":"false","result":"","error":"Data structure is corrupted"}
else:
c.result = {"success":"false","error":result["Message"]}
gLogger.info("\033[0;31mJOB SUBMIT REQUEST:\033[0m %s" % (time() - pagestart))
return c.result
def __getDirectories(self):
"""get the directories where to look into (they are either given, or taken from the transformation ID"""
if self.directories:
directories = []
printout = False
for directory in self.directories:
if not directory.endswith("..."):
directories.append(directory)
else:
printout = True
topDir = os.path.dirname(directory)
res = self.fileCatalog.listDirectory(topDir)
if not res["OK"]:
# DError(errno.ENOENT, res['Message'] )
return S_ERROR(errno.ENOENT, res["Message"])
else:
matchDir = directory.split("...")[0]
directories += [
d for d in res["Value"]["Successful"].get(
topDir, {}).get("SubDirs", [])
if d.startswith(matchDir)
]
if printout:
gLogger.always("Expanded list of %d directories:\n%s" %
(len(directories), "\n".join(directories)))
return directories
else:
return S_ERROR(errno.ENOENT, "Need to specify the directories")
def test_Gauss_Production_PR33857_1(self):
""" Using ProdConf (production style). systemConfig = 'x86_64-slc6-gcc48-opt'
This is taken from PR 33857 (and would fall back to SetupProject)
"""
gLogger.always("**** GAUSS v49r5")
ra = RunApplication()
ra.applicationName = 'Gauss'
ra.applicationVersion = 'v49r5'
ra.systemConfig = 'x86_64-slc6-gcc48-opt'
ra.commandOptions = ['$APPCONFIGOPTS/Gauss/Sim08-Beam4000GeV-mu100-2012-nu2.5.py',
'$APPCONFIGOPTS/Gauss/DataType-2012.py',
'$APPCONFIGOPTS/Gauss/RICHRandomHits.py',
'$APPCONFIGOPTS/Gauss/NoPacking.py',
'$DECFILESROOT/options/12877041.py',
'$LBPYTHIA8ROOT/options/Pythia8.py',
'$APPCONFIGOPTS/Gauss/G4PL_FTFP_BERT_EmNoCuts.py',
'$APPCONFIGOPTS/Persistency/Compression-ZLIB-1.py']
ra.extraPackages = [('AppConfig', 'v3r277'),
('Gen/DecFiles', 'v29r10'),
('ProdConf', '')
]
ra.prodConfFileName = find_all('test_prodConf_gauss_v49r5.py', '..')[0]
ra.applicationLog = '00033857_00000001_1_log.txt'
ra.stdError = '00033857_00000001_1_err.txt'
res = ra.run()
self.assertTrue(res['OK'])
self.assertEqual(res['Value'], (0, '', ''))
def web_getMetadataFilesInFile( self ):
self.set_header('Content-type','text/plain')
self.set_header('Content-Disposition', 'attachment; filename="error.txt"')
RPC = RPCClient( "DataManagement/FileCatalog" )
req = self.__request_file()
gLogger.always(req)
gLogger.debug( "submit: incoming request %s" % req )
result = yield self.threadTask(RPC.findFilesByMetadata, req["selection"] , req["path"])
if not result[ "OK" ] :
gLogger.error( "submit: %s" % result[ "Message" ] )
self.finish({ "success" : "false" , "error" : result[ "Message" ] })
return
result = result[ "Value" ]
retStrLines = []
if len(result)>0:
for key , value in result.items() :
for fileName in value:
retStrLines.append(key+"/"+fileName)
strData = "\n".join(retStrLines)
self.set_header('Content-type','text/plain')
self.set_header('Content-Disposition', 'attachment; filename="%s.txt"' % md5( str( req ) ).hexdigest())
self.set_header('Content-Length', len( strData ))
self.finish(strData)
def __commitConfiguration(self):
data = self.getSessionData()
isAuth = False
if "properties" in data["user"]:
if "CSAdministrator" in data["user"]["properties"]:
isAuth = True
if not isAuth:
return {
"success":
0,
"op":
"commitConfiguration",
"message":
"You are not authorized to commit configurations!! Bad boy!"
}
gLogger.always("User %s is commiting a new configuration version" %
data["user"]["DN"])
retDict = self.__configData['cfgData'].commit()
if not retDict['OK']:
return {
"success": 0,
"op": "commitConfiguration",
"message": retDict['Message']
}
return {"success": 1, "op": "commitConfiguration"}
def requestOptimization(self, job):
jid = job.getJobId()
gLogger.info("Sending job %d to optimizer %s" % (jid, self.__name))
self.__lock.acquire()
try:
if self.__frozen:
return S_ERROR("Optimizer is frozen")
if jid in self.__jobsInOptimizer:
gLogger.warn("Job %d already in optimizer %s" %
(jid, self.__name))
return S_OK()
if jid in self.__bookings:
del (self.__bookings[jid])
self.__jobsInOptimizer.add(jid)
finally:
self.__lock.release()
result = job.dumpToStub()
if not result['OK']:
return result
jobStub = result['Value']
gLogger.always("Sending job %s to %s [%s]" %
(jid, self.__name, self.__trid))
result = self.__msgSender.sendMessage(self.__trid, 'optimizeJob', jid,
jobStub)
if not result['OK']:
self.jobExitedOptimizer(jid)
return result
def printOperation(indexOperation, verbose=True, onlyFailed=False):
global output
i, op = indexOperation
prStr = ''
if op.SourceSE:
prStr += 'SourceSE: %s' % op.SourceSE
if op.TargetSE:
prStr += (' - ' if prStr else '') + 'TargetSE: %s' % op.TargetSE
if prStr:
prStr += ' - '
prStr += 'Created %s, Updated %s' % (op.CreationTime, op.LastUpdate)
if op.Type == 'ForwardDISET' and op.Arguments:
from DIRAC.Core.Utilities import DEncode
decode, _length = DEncode.decode(op.Arguments)
if verbose:
output = ''
prettyPrint(decode, offset=10)
prStr += '\n Arguments:\n' + output.strip('\n')
else:
prStr += '\n Service: %s' % decode[0][0]
gLogger.always(" [%s] Operation Type='%s' ID=%s Order=%s Status='%s'%s%s" %
(i, op.Type,
op.OperationID if hasattr(op, 'OperationID') else '(not set yet)',
op.Order, op.Status,
(" Error='%s'" % op.Error) if op.Error and op.Error.strip() else "",
(" Catalog=%s" % op.Catalog) if op.Catalog else ""))
if prStr:
gLogger.always(" %s" % prStr)
for indexFile in enumerate(op):
if not onlyFailed or indexFile[1].Status == 'Failed':
printFile(indexFile)
def test_DaVinci_new_gcc62(self):
""" Using ProdConf (production style). systemConfig = 'x86_64-slc6-gcc62-opt'
This is taken from step 130847
"""
gLogger.always("**** DAVINCI v42r2")
ra = RunApplication()
ra.applicationName = 'DaVinci'
ra.applicationVersion = 'v42r2'
ra.systemConfig = 'x86_64-slc6-gcc62-opt'
ra.commandOptions = ['$APPCONFIGOPTS/DaVinci/DV-Stripping27-Stripping.py',
'$APPCONFIGOPTS/DaVinci/DataType-2016.py',
'$APPCONFIGOPTS/DaVinci/InputType-RDST.py',
'$APPCONFIGOPTS/DaVinci/DV-RawEventJuggler-0_3-to-4_2.py']
ra.extraPackages = [('AppConfig', 'v3r308'),
('Det/SQLDDDB', 'v7r11'),
('ProdConf', '')
]
ra.step_Number = 1
ra.prodConfFileName = find_all('test_prodConf_davinci_v42r2.py', '..')[0]
ra.applicationLog = '0daVinci_000v42r2_62_log.txt'
ra.stdError = '0daVinci_000v42r2_62_err.txt'
res = ra.run()
self.assertTrue(res['OK'])
self.assertEqual(res['Value'], (0, '', ''))
def aftermath(self, actionSuccess, actionFailed, action, prefix):
success = ", ".join(actionSuccess)
failure = "\n".join(actionFailed)
if len(actionSuccess) > 1:
sText = prefix + "s"
else:
sText = prefix
if len(actionFailed) > 1:
fText = prefix + "s"
else:
fText = prefix
if len(success) > 0 and len(failure) > 0:
sMessage = "%s %sed successfully: " % (sText , action , success)
fMessage = "Failed to %s %s:\n%s" % (action , fText , failure)
result = sMessage + "\n\n" + fMessage
return { "success" : "true" , "result" : result }
elif len(success) > 0 and len(failure) < 1:
result = "%s %sed successfully: %s" % (sText , action , success)
return { "success" : "true" , "result" : result }
elif len(success) < 1 and len(failure) > 0:
result = "Failed to %s %s:\n%s" % (action , fText , failure)
gLogger.always(result)
return { "success" : "false" , "error" : result }
else:
result = "No action has performed due technical failure. Check the logs please"
gLogger.debug(result)
return { "success" : "false" , "error" : result }
def __getDirectories(self):
""" get the directories where to look into (they are either given, or taken from the transformation ID
"""
if self.directories:
directories = []
printout = False
for directory in self.directories:
if not directory.endswith('...'):
directories.append(directory)
else:
printout = True
topDir = os.path.dirname(directory)
res = self.fc.listDirectory(topDir)
if not res['OK']:
return S_ERROR(
errno.ENOENT, res['Message']
) #DError(errno.ENOENT, res['Message'] )
else:
matchDir = directory.split('...')[0]
directories += [
d for d in res['Value']['Successful'].get(
topDir, {}).get('SubDirs', [])
if d.startswith(matchDir)
]
if printout:
gLogger.always('Expanded list of %d directories:\n%s' %
(len(directories), '\n'.join(directories)))
return directories
else:
return S_ERROR(
errno.ENOENT, 'Need to specify the directories'
) #DError(errno.ENOENT, 'Need to specify the directories')
def insertApplicationToCS(self, name, csParameter):
"""add given application found via CVMFS to the CS"""
pars = dict(self.parameter)
pars['name'] = name
gLogger.notice("%(name)s: Adding version %(version)s to the CS" % pars)
existingVersions = gConfig.getSections("%(softSec)s/%(platform)s/%(name)s" % pars, [])
if not existingVersions['OK']:
gLogger.error("Could not find all versions available in CS: %s" % existingVersions['Message'])
dexit(255)
if pars['version'] in existingVersions['Value']:
gLogger.always('Application %s %s for %s already in CS, nothing to do' % (name.lower(),
pars['version'],
pars['platform']))
return S_OK()
csPath = self.softSec + ("/%(platform)s/%(name)s/%(version)s/" % pars)
for par, val in csParameter.iteritems():
gLogger.notice("Add: %s = %s" %(csPath+par, val))
result = self.csAPI.setOption(csPath+par, val)
if result['OK']:
self.modifiedCS = True
else:
gLogger.error("Failure to add to CS", result['Message'])
return S_ERROR("")
return S_OK()
def __request( self ):
gLogger.info( "!!! PARAMS: ", str( self.request.arguments ) )
req = {}
start = 0
limit = 25
if self.request.arguments.has_key( "limit" ) and len( self.request.arguments["limit"][0] ) > 0:
limit = int( self.request.arguments["limit"][0] )
if self.request.arguments.has_key( "start" ) and len( self.request.arguments["start"][0] ) > 0:
start = int( self.request.arguments["start"][0] )
try:
sortDirection = str( self.request.arguments[ 'sortDirection' ] ).strip()
except:
sortDirection = "ASC"
try:
sortField = str( self.request.arguments[ 'sortField' ] ).strip()
except:
sortField = "UserName"
sort = [[sortField, sortDirection]]
gLogger.info( "!!! S O R T : ", sort )
if "username" in self.request.arguments:
users = list( json.loads( self.request.arguments[ 'username' ][-1] ) )
if len( users ) > 0:
req['UserName'] = users
if "usergroup" in self.request.arguments:
usersgroup = list( json.loads( self.request.arguments[ 'usergroup' ][-1] ) )
if len( usersgroup ) > 0:
req['UserGroup'] = usersgroup
if "usersgroup" in self.request.arguments and len( self.request.arguments["persistent"] ) > 0:
if str( self.request.arguments["persistent"] ) in ["True", "False"]:
req["PersistentFlag"] = str( self.request.arguments["persistent"] )
before = False
after = False
if self.request.arguments.has_key( "expiredBefore" ) and len( self.request.arguments["expiredBefore"] ) > 0:
try:
before = int( self.request.arguments["expiredBefore"] )
except:
pass
if self.request.arguments.has_key( "expiredAfter" ) and len( self.request.arguments["expiredAfter"] ) > 0:
try:
after = int( self.request.arguments["expiredAfter"] )
except:
pass
if before and after:
if before > after:
req["beforeDate"] = before
req["afterDate"] = after
else:
if before:
req["beforeDate"] = before
if after:
req["afterDate"] = after
gLogger.always( "REQUEST:", req )
return ( start, limit, sort, req )
def run(self):
""" The main watchdog execution method
"""
result = self.initialize()
if not result["OK"]:
gLogger.always("Can not start watchdog for the following reason")
gLogger.always(result["Message"])
return result
try:
while True:
gLogger.debug("Starting watchdog loop # %d" % self.count)
start_cycle_time = time.time()
result = self.execute()
exec_cycle_time = time.time() - start_cycle_time
if not result["OK"]:
gLogger.error("Watchdog error during execution", result["Message"])
break
elif result["Value"] == "Ended":
break
self.count += 1
if exec_cycle_time < self.pollingTime:
time.sleep(self.pollingTime - exec_cycle_time)
return S_OK()
except Exception:
gLogger.exception()
return S_ERROR("Exception")
def run(self):
""" The main watchdog execution method
"""
result = self.initialize()
if not result['OK']:
gLogger.always('Can not start wtchdog for the following reason')
gLogger.always(result['Message'])
return result
try:
while True:
gLogger.debug('Starting agent loop # %d' % self.count)
start_cycle_time = time.time()
result = self.execute()
exec_cycle_time = time.time() - start_cycle_time
if not result['OK']:
gLogger.error("Watchdog error during execution",
result['Message'])
break
elif result['Value'] == "Ended":
break
self.count += 1
if exec_cycle_time < self.pollingTime:
time.sleep(self.pollingTime - exec_cycle_time)
return S_OK()
except Exception, x:
gLogger.exception()
return S_ERROR('Exception')
def serve(self):
result = self.__createListeners()
if not result['OK']:
self.__closeListeningConnections()
return result
for svcName in self.__listeningConnections:
gLogger.always("Listening at %s" % self.__services[svcName].getConfig().getURL())
isMultiProcessingAllowed = False
for svcName in self.__listeningConnections:
if self.__services[svcName].getConfig().getCloneProcesses() > 0:
isMultiProcessingAllowed = True
break
if isMultiProcessingAllowed:
signal.signal(signal.SIGTERM, self.stopChildProcesses)
signal.signal(signal.SIGINT, self.stopChildProcesses)
for svcName in self.__listeningConnections:
clones = self.__services[svcName].getConfig().getCloneProcesses()
for i in range(1, clones):
p = multiprocessing.Process(target=self.__startCloneProcess, args=(svcName, i))
self.__processes.append(p)
p.start()
gLogger.always("Started clone process %s for %s" % (i, svcName))
while self.__alive:
self.__acceptIncomingConnection()
def getOption(self, switch, default=None):
"""
Get a specific items set by the setters
"""
if switch == 'SEs':
# SEs have to be resolved recursively using StorageElementGroups
return resolveSEGroup(self.options.get(switch, default))
value = self.options.get(switch, default)
if switch in ('LFNs', 'Directory'):
# Special case for getting LFNs or directories: parse the option
if value == default and switch == 'Directory':
value = self.options.get('LFNs', default)
if not value:
if not sys.stdin.isatty():
# If the input file is a pipe, no need to specify it
self.setLFNsFromTerm()
value = self.options.get('LFNs', default)
if value:
# Parse the LFNs out of the "LFNs" option list
value = self.getLFNsFromList(value,
directories=switch == 'Directory')
if value and self.setLastFile and switch == 'LFNs':
# Storethe list of LFNs in a temporary file
gLogger.always("Got %d LFNs" % len(value))
if self.setLastFile != self.lastFile:
self.setLastFile = False
with open(self.lastFile, 'w') as tmpFile:
tmpFile.write('\n'.join(sorted(value)))
if isinstance(value, set):
# Return a sorted list from a set
value = sorted(value)
return value
def printFile( indexFile ):
j, f = indexFile
gLogger.always( " [%02d] ID=%s LFN='%s' Status='%s'%s%s%s" % ( j + 1, f.FileID if hasattr( f, 'FileID' ) else '(not set yet)', f.LFN, f.Status,
( " Checksum='%s'" % f.Checksum ) if f.Checksum or ( f.Error and 'checksum' in f.Error.lower() ) else "",
( " Error='%s'" % f.Error ) if f.Error and f.Error.strip() else "",
( " Attempts=%d" % f.Attempt ) if f.Attempt > 1 else ""
)
)
def printFile( indexFile ):
j, f = indexFile
gLogger.always( " [%02d] ID=%s LFN='%s' Status='%s'%s%s%s" % ( j + 1, f.FileID if hasattr( f, 'FileID' ) else '(not set yet)', f.LFN, f.Status,
( " Checksum='%s'" % f.Checksum ) if f.Checksum or ( f.Error and 'checksum' in f.Error.lower() ) else "",
( " Error='%s'" % f.Error ) if f.Error and f.Error.strip() else "",
( " Attempts=%d" % f.Attempt ) if f.Attempt > 1 else ""
)
)
def __request(self):
gLogger.info("!!! PARAMS: ",str(request.params))
req = {}
try:
start = int( request.params[ 'start' ] )
except:
start = 0
try:
limit = int( request.params[ 'limit' ] )
except:
limit = 25
try:
sortDirection = str( request.params[ 'sortDirection' ] ).strip()
except:
sortDirection = "ASC"
try:
sortField = str( request.params[ 'sortField' ] ).strip()
except:
sortField = "UserName"
sort = [[sortField, sortDirection]]
gLogger.info("!!! S O R T : ",sort)
result = gConfig.getOption("/Website/ListSeparator")
if result["OK"]:
separator = result["Value"]
else:
separator = ":::"
if request.params.has_key("username") and len(request.params["username"]) > 0:
if str(request.params["username"]) != "All":
req["UserName"] = str(request.params["username"]).split(separator)
if request.params.has_key("usergroup") and len(request.params["usergroup"]) > 0:
if str(request.params["usergroup"]) != "All":
req["UserGroup"] = str(request.params["usergroup"]).split(separator)
if request.params.has_key("persistent") and len(request.params["persistent"]) > 0:
if str(request.params["persistent"]) in ["True","False"]:
req["PersistentFlag"] = str(request.params["persistent"])
before = False
after = False
if request.params.has_key("expiredBefore") and len(request.params["expiredBefore"]) > 0:
try:
before = int(request.params["expiredBefore"])
except:
pass
if request.params.has_key("expiredAfter") and len(request.params["expiredAfter"]) > 0:
try:
after = int(request.params["expiredAfter"])
except:
pass
if before and after:
if before > after:
req["beforeDate"] = before
req["afterDate"] = after
else:
if before:
req["beforeDate"] = before
if after:
req["afterDate"] = after
gLogger.always("REQUEST:",req)
return (start, limit, sort, req)
def run( self ):
port = RESTConf.port()
if self.__sslops:
url = "https://0.0.0.0:%s" % port
else:
url = "http://0.0.0.0:%s" % port
gLogger.always( "Starting REST server on %s" % url )
autoreload.add_reload_hook( self.__reloadAppCB )
ioloop.IOLoop.instance().start()
def web_hostAction( self ):
"""
Restart all DIRAC components on a given host
"""
if not "host" in self.request.arguments:
self.finish( { "success" : "false" , "error" : "No hostname defined" } )
return
if not "action" in self.request.arguments:
self.finish( { "success" : "false" , "error" : "No action defined" } )
return
action = str( self.request.arguments[ "action" ][0] )
hosts = self.request.arguments[ "host" ][0].split( "," )
version = self.request.arguments[ "version" ][0]
userData = self.getSessionData()
DN = str( userData["user"]["DN"] )
group = str( userData["user"]["group"] )
actionSuccess = list()
actionFailed = list()
for i in hosts:
client = SystemAdministratorClient( str( i ) , None , delegatedDN = DN ,
delegatedGroup = group )
if action == "restart":
result = yield self.threadTask( client.restartComponent, str( "*" ) , str( "*" ) )
elif action == "revert":
result = yield self.threadTask( client.revertSoftware )
elif action == "update":
result = yield self.threadTask( client.updateSoftware, version, '', '', timeout = 300 )
else:
error = i + ": Action %s is not defined" % action
actionFailed.append( error )
continue
gLogger.always( result )
if not result[ "OK" ]:
if result[ "Message" ].find( "Unexpected EOF" ) > 0:
msg = "Signal 'Unexpected EOF' received: %s. Most likely DIRAC components" % result['Message']
msg = i + ": " + msg + " were successfully restarted."
actionSuccess.append( msg )
continue
error = i + ": " + result[ "Message" ]
actionFailed.append( error )
gLogger.error( error )
else:
gLogger.info( result[ "Value" ] )
actionSuccess.append( i )
self.finish( self.aftermath( actionSuccess, actionFailed, action, "Host" ) )
def commitConfiguration(self):
if not authorizeAction():
return S_ERROR("You are not authorized to commit configurations!! Bad boy!")
gLogger.always("User %s is commiting a new configuration version" % credentials.getUserDN())
modifier = self.__getModificator()
modifier.loadFromBuffer(session["cfgData"])
retDict = modifier.commit()
if not retDict["OK"]:
return S_ERROR(retDict["Message"])
return S_OK()
def showVersion(self, _arg):
""" Show version
:param _arg: unuse
:return: S_OK()
"""
gLogger.always("Version: %s" % __RCSID__)
sys.exit(0)
return S_OK()
def printFTSJobs(request):
""" Prints the FTSJobs associated to a request
:param request: Request object
"""
try:
if request.RequestID:
# We try first the new FTS3 system
from DIRAC.DataManagementSystem.Client.FTS3Client import FTS3Client
fts3Client = FTS3Client()
res = fts3Client.ping()
if res['OK']:
associatedFTS3Jobs = []
for op in request:
res = fts3Client.getOperationsFromRMSOpID(op.OperationID)
if res['OK']:
for fts3Op in res['Value']:
associatedFTS3Jobs.extend(fts3Op.ftsJobs)
if associatedFTS3Jobs:
gLogger.always(
'\n\nFTS3 jobs associated: \n%s' %
'\n'.join('%s@%s (%s)' %
(job.ftsGUID, job.ftsServer, job.status)
for job in associatedFTS3Jobs))
return
# If we are here, the attempt with the new FTS3 system did not work, let's try the old FTS system
gLogger.debug("Could not instantiate FTS3Client", res)
from DIRAC.DataManagementSystem.Client.FTSClient import FTSClient
ftsClient = FTSClient()
res = ftsClient.ping()
if not res['OK']:
gLogger.debug("Could not instantiate FtsClient", res)
return
res = ftsClient.getFTSJobsForRequest(request.RequestID)
if res['OK']:
ftsJobs = res['Value']
if ftsJobs:
gLogger.always(' FTS jobs associated: %s' %
','.join('%s (%s)' %
(job.FTSGUID, job.Status)
for job in ftsJobs))
# ImportError can be thrown for the old client
# AttributeError can be thrown because the deserialization will not have
# happened correctly on the new fts3 (CC7 typically), and the error is not
# properly propagated
except (ImportError, AttributeError) as err:
gLogger.debug("Could not instantiate FtsClient because of Exception",
repr(err))
def main():
global includeMasterCS
Script.registerSwitch("n", "noMasterCS", "do not include master CS",
setNoMasterCS)
Script.parseCommandLine()
from DIRAC import gLogger, exit as DIRACExit
from DIRAC.WorkloadManagementSystem.Utilities.PilotCStoJSONSynchronizer import PilotCStoJSONSynchronizer
ps = PilotCStoJSONSynchronizer()
gLogger.verbose("Parameters for this sync:")
gLogger.verbose("repo=" + ps.pilotRepo)
gLogger.verbose("VO repo=" + ps.pilotVORepo)
gLogger.verbose("projectDir=" + ps.projectDir)
gLogger.verbose("pilotScriptsPath=" + ps.pilotScriptPath)
gLogger.verbose("pilotVOScriptsPath=" + ps.pilotVOScriptPath)
gLogger.verbose("pilotRepoBranch=" + ps.pilotRepoBranch)
gLogger.verbose("pilotVORepoBranch=" + ps.pilotVORepoBranch)
# pilot.json
res = ps.getCSDict(includeMasterCS=includeMasterCS)
if not res['OK']:
DIRACExit(1)
pilotDict = res['Value']
print(json.dumps(
pilotDict, indent=4,
sort_keys=True)) # just print here as formatting is important
with open('pilot.json', 'w') as jf:
json.dump(pilotDict, jf)
# pilot files
res = ps.syncScripts()
if not res['OK']:
DIRACExit(1)
gLogger.always(res['Value'])
tarPath, tarFiles = res['Value']
allFiles = [tarPath] + tarFiles + ['pilot.json']
# checksums
checksumDict = {}
for pFile in allFiles:
filename = os.path.basename(pFile)
with open(pFile, 'rb') as fp:
checksumDict[filename] = hashlib.sha512(fp.read()).hexdigest()
cksPath = 'checksums.sha512'
with open(cksPath, 'wt') as chksums:
for filename, chksum in sorted(checksumDict.items()):
# same as the output from sha512sum commands
chksums.write('%s %s\n' % (chksum, filename))
allFiles = allFiles + [cksPath]
print(allFiles)
def printFTSJobs(request):
""" Prints the FTSJobs associated to a request
:param request: Request object
"""
try:
if request.RequestID:
# We try first the new FTS3 system
from DIRAC.DataManagementSystem.Client.FTS3Client import FTS3Client
fts3Client = FTS3Client()
res = fts3Client.ping()
if res['OK']:
associatedFTS3Jobs = []
for op in request:
res = fts3Client.getOperationsFromRMSOpID(op.OperationID)
if res['OK']:
for fts3Op in res['Value']:
associatedFTS3Jobs.extend(fts3Op.ftsJobs)
if associatedFTS3Jobs:
gLogger.always(
'\n\nFTS3 jobs associated: \n%s' %
'\n'.join(
'%s@%s (%s)' %
(job.ftsGUID,
job.ftsServer,
job.status) for job in associatedFTS3Jobs))
return
# If we are here, the attempt with the new FTS3 system did not work, let's try the old FTS system
gLogger.debug("Could not instantiate FTS3Client", res)
from DIRAC.DataManagementSystem.Client.FTSClient import FTSClient
ftsClient = FTSClient()
res = ftsClient.ping()
if not res['OK']:
gLogger.debug("Could not instantiate FtsClient", res)
return
res = ftsClient.getFTSJobsForRequest(request.RequestID)
if res['OK']:
ftsJobs = res['Value']
if ftsJobs:
gLogger.always(' FTS jobs associated: %s' % ','.join('%s (%s)' % (job.FTSGUID, job.Status)
for job in ftsJobs))
# ImportError can be thrown for the old client
# AttributeError can be thrown because the deserialization will not have
# happened correctly on the new fts3 (CC7 typically), and the error is not
# properly propagated
except (ImportError, AttributeError) as err:
gLogger.debug("Could not instantiate FtsClient because of Exception", repr(err))
def __getSelector(self,select="All"):
RPC = getRPCClient("DataManagement/FileCatalog")
result = RPC.getMetadataFields()
if not result["OK"]:
return {"success":"false","error":result["Message"]}
result = result["Value"]
gLogger.always(" * * * ",result)
for key,value in result.items():
result[key] = value.lower()
gLogger.always(" * * * ",result)
return {"success":"true","result":result}
def __getStats(self,selector):
gLogger.always(" --- selector : %s" % selector)
# import sys
# sys.stdout.flush()
req = self.__request()
selector = str(selector)
RPC = getRPCClient("WorkloadManagement/JobMonitoring")
if selector == "Minor status":
selector = "MinorStatus"
elif selector == "Application status":
selector = "ApplicationStatus"
gLogger.always(" --- getJobStats(%s,%s) : " % (str(selector),str(req)))
result = RPC.getJobStats(selector,req)
if result["OK"]:
c.result = []
result = dict(result["Value"])
keylist = result.keys()
keylist.sort()
if selector == "Site":
tier1 = gConfig.getValue("/Website/PreferredSites")
if tier1:
try:
tier1 = tier1.split(", ")
tier1.sort()
except:
tier1 = False
else:
tier1 = False
if tier1 and len(tier1) > 0:
for i in tier1:
if result.has_key(i):
countryCode = i.rsplit(".",1)[1]
c.result.append({"Key":i,"Value":result[i],"Code":countryCode})
for key in keylist:
if selector == "Site" and tier1:
if key not in tier1:
try:
countryCode = key.rsplit(".",1)[1]
except:
countryCode = "Unknown"
c.result.append({"Key":key,"Value":result[key],"Code":countryCode})
elif selector == "Site" and not tier1:
try:
countryCode = key.rsplit(".",1)[1]
except:
countryCode = "Unknown"
c.result.append({"Key":key,"Value":result[key],"Code":countryCode})
else:
c.result.append({"Key":key,"Value":result[key]})
c.result = {"success":"true","result":c.result}
else:
c.result = {"success":"false","error":result["Message"]}
return c.result
def mergeCallback(fileList, mergePath, mergeSize, mergeSpan, ret):
status = 'ok' if ret else 'error'
size = mergeSize / (1024*1024.)
if status not in mergeCounter:
mergeCounter[status] = 0
mergeCounter[status] += 1
mergeCounter['total'] += 1
mergeSpeed['size'] += size
mergeSpeed['span'] += mergeSpan
gLogger.always('[Merged] %-8s %9.2f MB %6d files %s' % (status, size, len(fileList), mergePath))
def __commitChanges(self):
data = self.getSessionData()
isAuth = False
if "properties" in data["user"]:
if "CSAdministrator" in data["user"]["properties"]:
isAuth = True
if not isAuth:
return {"success":0, "op":"commitChanges", "message":"You are not authorized to commit changes!!"}
gLogger.always("User %s is commiting a new configuration version" % data["user"]["DN"])
retDict = self.__configData[ 'cfgData' ].commit()
if not retDict[ 'OK' ]:
return {"success":0, "op":"commitChanges", "message":retDict[ 'Message' ]}
return {"success":1, "op":"commitChanges"}
def printRequest(request, status=None, full=False, verbose=True, terse=False):
global output
if full:
output = ''
prettyPrint(json.loads(request.toJSON()['Value']))
gLogger.always(output)
else:
if not status:
status = request.Status
gLogger.always("Request name='%s' ID=%s Status='%s'%s%s%s" %
(request.RequestName,
request.RequestID if hasattr(request, 'RequestID') else '(not set yet)',
request.Status, " ('%s' in DB)" % status if status != request.Status else '',
(" Error='%s'" % request.Error) if request.Error and request.Error.strip() else "",
(" Job=%s" % request.JobID) if request.JobID else ""))
gLogger.always("Created %s, Updated %s%s" % (request.CreationTime,
request.LastUpdate,
(", NotBefore %s" % request.NotBefore) if request.NotBefore else ""))
if request.OwnerDN:
gLogger.always("Owner: '%s', Group: %s" % (request.OwnerDN, request.OwnerGroup))
for indexOperation in enumerate(request):
op = indexOperation[1]
if not terse or op.Status == 'Failed':
printOperation(indexOperation, verbose, onlyFailed=terse)
printFTSJobs(request)
def printFTSJobs(request):
""" Prints the FTSJobs associated to a request
:param request: Request object
"""
try:
if request.RequestID:
# We try first the new FTS3 system
from DIRAC.DataManagementSystem.Client.FTS3Client import FTS3Client
fts3Client = FTS3Client()
res = fts3Client.ping()
if res['OK']:
associatedFTS3Jobs = []
for op in request:
res = fts3Client.getOperationsFromRMSOpID(op.OperationID)
if res['OK']:
for fts3Op in res['Value']:
associatedFTS3Jobs.extend(fts3Op.ftsJobs)
if associatedFTS3Jobs:
gLogger.always(
'\n\nFTS3 jobs associated: \n%s' %
'\n'.join(
'%s@%s (%s)' %
(job.ftsGUID,
job.ftsServer,
job.status) for job in associatedFTS3Jobs))
return
# If we are here, the attempt with the new FTS3 system did not work, let's try the old FTS system
gLogger.debug("Could not instantiate FTS3Client", res)
from DIRAC.DataManagementSystem.Client.FTSClient import FTSClient
ftsClient = FTSClient()
res = ftsClient.ping()
if not res['OK']:
gLogger.debug("Could not instantiate FtsClient", res)
return
res = ftsClient.getFTSJobsForRequest(request.RequestID)
if res['OK']:
ftsJobs = res['Value']
if ftsJobs:
gLogger.always(' FTS jobs associated: %s' % ','.join('%s (%s)' % (job.FTSGUID, job.Status)
for job in ftsJobs))
except ImportError as err:
gLogger.debug("Could not instantiate FtsClient because of Exception", repr(err))
def __prepareURL(self,files):
files = files.split(",")
if not len(files) > 0:
return {"success":"false","error":"No LFN given"}
se = getRPCClient("DataManagement/StorageElementProxy")
result = se.prepareFileForHTTP(files)
gLogger.always(" *** ",result)
if not result["OK"]:
return {"success":"false","error":result["Message"]}
httpURLs = result['HttpURL']
httpKey = result['HttpKey']
return {"success":"true","result":{"url":httpURLs,"cookie":httpKey}}
def submit(self):
RPC = getRPCClient("DataManagement/FileCatalog")
req = self.__request()
result = RPC.findFilesByMetadata(req,"/")
gLogger.always(" - REZ: %s" % result)
if not result["OK"]:
return {"success":"false","error":result["Message"]}
result = result["Value"]
if not len(result) > 0:
return {"success":"true","result":{},"total":0}
callback = list()
for key,value in result.items():
if len(value) > 0:
for j in value:
callback.append({"filename":key + "/" + j})
return {"success":"true","result":callback,"total":len(callback)}
def serve( self ):
result = self.__createListeners()
if not result[ 'OK' ]:
self.__closeListeningConnections()
return result
for svcName in self.__listeningConnections:
gLogger.always( "Listening at %s" % self.__services[ svcName ].getConfig().getURL() )
#Multiple clones not yet working. Disabled by default
if False and multiprocessing:
for svcName in self.__listeningConnections:
clones = self.__services[ svcName ].getConfig().getCloneProcesses()
for i in range( 1, clones ):
p = multiprocessing.Process( target = self.__startCloneProcess, args = ( svcName, i ) )
p.start()
gLogger.always( "Started clone process %s for %s" % ( i, svcName ) )
while self.__alive:
self.__acceptIncomingConnection()
def __actionHost( self ):
"""
Restart all DIRAC components on a given host
"""
if not "hostname" in request.params:
return { "success" : "false" , "error" : "No hostname given" }
hosts = request.params[ "hostname" ].split( "," )
DN = getUserDN()
group = getSelectedGroup()
self.actionSuccess = list()
self.actionFailed = list()
for i in hosts:
client = SystemAdministratorClient( str( i ) , None , delegatedDN=DN ,
delegatedGroup=group )
if self.action is "restart":
result = client.restartComponent( str( "*" ) , str( "*" ) )
elif self.action is "revert":
result = client.revertSoftware()
else:
error = i + ": Action %s is not defined" % self.action
self.actionFailed.append( error )
continue
gLogger.always( result )
if not result[ "OK" ]:
if result[ "Message" ].find( "Unexpected EOF" ) > 0:
msg = "Signal 'Unexpected EOF' received. Most likely DIRAC components"
msg = i + ": " + msg + " were successfully restarted."
self.actionSuccess.append( msg )
continue
error = i + ": " + result[ "Message" ]
self.actionFailed.append( error )
gLogger.error( error )
else:
gLogger.info( result[ "Value" ] )
self.actionSuccess.append( i )
self.prefix = "Host"
return self.__aftermath()
def __getOptionsFromCS( self , path = "/Website/Launchpad/Options" , delimiter = "," ):
gLogger.info( "start __getOptionsFromCS" )
result = gConfig.getOptionsDict( path )
gLogger.always( result )
if not result["OK"]:
return False
options = result["Value"]
for i in options.keys():
options[ i ] = options[ i ].split( delimiter )
result = gConfig.getSections(path)
if result["OK"]:
sections = result["Value"]
if len(sections) > 0:
for i in sections:
options[ i ] = self.__getOptionsFromCS( path + '/' + i , delimiter )
gLogger.always( "options: %s" % options )
gLogger.info( "end __getOptionsFromCS" )
return options
def web_getFilesData( self ) :
RPC = RPCClient( "DataManagement/FileCatalog", timeout=3600 )
req = self.__request()
gLogger.always(req)
gLogger.debug( "submit: incoming request %s" % req )
result = yield self.threadTask(RPC.findFilesByMetadataWeb, req["selection"] , req["path"] , self.S_NUMBER , self.L_NUMBER)
gLogger.debug( "submit: result of findFilesByMetadataDetailed %s" % result )
if not result[ "OK" ] :
gLogger.error( "submit: %s" % result[ "Message" ] )
self.finish({ "success" : "false" , "error" : result[ "Message" ] })
return
result = result[ "Value" ]
if not len(result) > 0:
self.finish({ "success" : "true" , "result" : [] , "total" : 0, "date":"-" })
return
total = result[ "TotalRecords" ]
result = result[ "Records" ]
callback = list()
for key , value in result.items() :
size = ""
if "Size" in value:
size = value[ "Size" ]
date = ""
if "CreationDate" in value:
date = str( value[ "CreationDate" ] )
meta = ""
if "Metadata" in value:
m = value[ "Metadata" ]
meta = '; '.join( [ '%s: %s' % ( i , j ) for ( i , j ) in m.items() ] )
dirnameList = key.split("/")
dirname = "/".join(dirnameList[:len(dirnameList)-1])
filename = dirnameList[len(dirnameList)-1:]
callback.append({"fullfilename":key, "dirname": dirname, "filename" : filename , "date" : date , "size" : size ,
"metadata" : meta })
timestamp = Time.dateTime().strftime("%Y-%m-%d %H:%M [UTC]")
self.finish({ "success" : "true" , "result" : callback , "total" : total, "date":timestamp})