def setUp(self):
RESTBaseUnitTest.setUp(self)
self.testInit.setupCouch("%s" % self.couchDBName, "GroupUser",
"ConfigCache", "ReqMgr")
self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats")
# logging stuff from TestInit is broken, setting myself
l = logging.getLogger()
l.setLevel(logging.DEBUG)
self.params = {}
self.params['endpoint'] = self.config.getServerUrl()
self.reqService = RequestManagerDS(self.params)
self.jsonSender = JSONRequests(self.config.getServerUrl())
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName=userName,
groupName=groupName,
teamName=teamName)
schema['ConfigCacheID'] = self.createConfig()
schema['CouchDBName'] = self.couchDBName
schema['CouchWorkloadDBName'] = self.couchDBName
try:
r = self.jsonSender.put('request', schema)
try:
self.requestName = r[0]['RequestName']
except:
self.requestName = r[0].values()[0]['RequestName']
except Exception as ex:
msg = traceback.format_exc()
print "Exception during set up, reason: %s" % msg
raise ex
def decodeJson(self, result):
"""
decodeJson
decode the response result reveiced from the server
"""
encoder = JSONRequests()
return encoder.decode(result)
def decodeJson(self, result):
"""
decodeJson
decode the response result reveiced from the server
"""
encoder = JSONRequests(idict={"pycurl": True})
return encoder.decode(result)
def wmbsServiceSetup(self, argstring, kargs={}, returnType='text'):
if returnType == 'json':
request = JSONRequests(self.server_url)
else:
request = Requests(self.server_url)
results = request.get("/wmbs/%s/" % argstring, kargs)
return results
def makeRequest(self, uri = None, data = None, verb = 'GET'):
"""
Make a request to the remote database. for a give URI. The type of
request will determine the action take by the server (be careful with
DELETE!). Data should be a dictionary of {dataname: datavalue}.
Returns a tuple of the data from the server, decoded using the
appropriate method the response status and the response reason, to be
used in error handling.
You can override the method to encode/decode your data by passing in an
encoding/decoding function to this method. Your encoded data must end up
as a string.
"""
data = data or {}
headers = {
"User-agent": "CRABClient/%s" % self['version'],
"Accept": "*/*",
}
#Quoting the uri since it can contain the request name, and therefore spaces (see #2557)
uri = urllib.quote(uri)
caCertPath = self.getCACertPath()
url = 'https://' + self['host'] + uri
#retries this up to self['retry'] times a range of exit codes
for i in range(self['retry'] + 1):
try:
response, datares = self['conn'].request(url, data, encode=True, headers=headers, verb=verb, doseq = True,
ckey=self['key'], cert=self['cert'], capath=caCertPath,
verbose=self['verbose'])
except Exception as ex:
#add here other temporary errors we need to retry
if (not retriableError(ex)) or (i == self['retry']):
msg = "Fatal error trying to connect to %s using %s" % (url, data)
self.logger.error(msg)
raise #really exit and raise exception if this was the last retry or the exit code is not among the list of the one we retry
sleeptime = 20 * (i + 1)
msg = "Sleeping %s seconds after HTTP error. Error details: " % sleeptime
if hasattr(ex, 'headers'):
msg += str(ex.headers)
else:
msg += str(ex)
self.logger.debug(msg)
time.sleep(sleeptime) #sleeps 20s the first time, 40s the second time and so on
else:
break
try:
result = JSONRequests(idict={"pycurl" : True}).decode(datares)
except Exception as ex:
msg = "Fatal error reading data from %s using %s" % (url, data)
self.logger.error(msg)
raise #really exit and raise exception
return result, response.status, response.reason
def __init__(self, cfg, statedir, testName):
self.server = RESTMain(cfg, statedir)
self.testName = testName
self.cofig = cfg
self.port = cfg.main.port
self.host = '127.0.0.1'
self.serverUrl = "http://%s:%s/%s/" % (self.host, self.port, cfg.main.application)
## test permission
#test_authz_key = fake_authz_key_file()
#self.header = fake_authz_headers(test_authz_key.data, roles = {"Global Admin": {'group': ['global']}})
self.jsonSender = JSONRequests(self.serverUrl)
def setUp(self):
"""
setUP global values
Database setUp is done in base class
"""
self.couchDBName = "reqmgr_t_0"
RESTBaseUnitTest.setUp(self)
self.testInit.setupCouch("%s" % self.couchDBName, "ConfigCache",
"ReqMgr")
reqMgrHost = self.config.getServerUrl()
self.jsonSender = JSONRequests(reqMgrHost)
def _verifyDBSCall(dbsURL, uri):
try:
# from WMCore.Services.DBS.DBS3Reader import DBS3Reader
# DBS3Reader(dbsURL).dbs.serverinfo()
from WMCore.Services.Requests import JSONRequests
jsonSender = JSONRequests(dbsURL)
result = jsonSender.get("/%s" % uri)
if not result[1] == 200:
raise WMSpecFactoryException("DBS is not connected: %s : %s" % (dbsURL, str(result)))
except:
raise WMSpecFactoryException("DBS is not responding: %s" % dbsURL)
return result[0]
def __init__(self, name, location, datasetpath):
"""
Just the necessary objects
Expects:
name: The blockname in full
location: The PNN of the site the block is at
"""
self.data = {
'dataset_conf_list': [], # List of dataset configurations
'file_conf_list': [], # List of files, the configuration for each
'files': [], # List of file objects
'block': {}, # Dict of block info
'processing_era': {}, # Dict of processing era info
'acquisition_era': {}, # Dict of acquisition era information
'primds': {}, # Dict of primary dataset info
'dataset': {}, # Dict of processed dataset info
'file_parent_list': [], # List of file parents
'dataset_parent_list':
[], # List of parent datasets (DBS requires this as list although it only allows one parent)
'close_settings': {}
} # Dict of info about block close settings
self.files = []
self.encoder = JSONRequests()
self.status = 'Open'
self.inBuff = False
self.startTime = time.time()
self.name = name
self.location = location
self.datasetpath = datasetpath
self.workflows = set()
self.data['block']['block_name'] = name
self.data['block']['origin_site_name'] = location
self.data['block']['open_for_writing'] = 1
self.data['block']['create_by'] = "WMAgent"
self.data['block']['creation_date'] = int(time.time())
self.data['block']['block_size'] = 0
self.data['block']['file_count'] = 0
self.data['block']['block_events'] = 0
self.data['close_settings'] = {}
self.data['close_settings']['block_close_max_wait_time'] = None
self.data['close_settings']['block_close_max_events'] = None
self.data['close_settings']['block_close_max_size'] = None
self.data['close_settings']['block_close_max_files'] = None
return
def __init__(self, cfg, statedir, testName):
self.server = RESTMain(cfg, statedir)
self.testName = testName
self.config = cfg
self.port = cfg.main.port
self.host = '127.0.0.1'
self.serverUrl = "http://%s:%s/%s/" % (self.host, self.port, cfg.main.application)
## test authentication using fake filepermission
self.test_authz_key = fake_authz_key_file(False)
self.config.main.tools.cms_auth.key_file = self.test_authz_key.name
#self.header = fake_authz_headers(test_authz_key.data, roles = {"Admin": {'group': ['ReqMgr']}})
self.jsonSender = JSONRequests(self.serverUrl)
def setUp(self):
"""
setUP global values
Database setUp is done in base class
"""
self.couchDBName = "reqmgr_t_0"
RESTBaseUnitTest.setUp(self)
self.testInit.setupCouch("%s" % self.couchDBName, "ConfigCache", "ReqMgr")
self.testInit.setupCouch("%s_wmstats" % self.couchDBName,
"WMStats")
self.testInit.setupCouch("%s_acdc" % self.couchDBName,
"ACDC", "GroupUser")
reqMgrHost = self.config.getServerUrl()
self.jsonSender = JSONRequests(reqMgrHost)
self.params = {}
self.params['endpoint'] = reqMgrHost
self.reqService = RequestManager(self.params)
def __init__(self, name, location, das):
"""
Just the necessary objects
Expects:
name: The blockname in full
location: The SE-name of the site the block is at
"""
self.data = {
'dataset_conf_list': [], # List of dataset configurations
'file_conf_list': [], # List of files, the configuration for each
'files': [], # List of file objects
'block': {}, # Dict of block info
'processing_era': {}, # Dict of processing era info
'acquisition_era': {}, # Dict of acquisition era information
'primds': {}, # Dict of primary dataset info
'dataset': {}, # Dict of processed dataset info
'file_parent_list': []
} # List of file parents
self.files = []
self.encoder = JSONRequests()
self.status = 'Open'
self.inBuff = False
self.startTime = time.time()
self.name = name
self.location = location
self.das = das
self.data['block']['block_name'] = name
self.data['block']['origin_site_name'] = location
self.data['block'][
'open_for_writing'] = 0 # If we're sending a block, it better be open
self.data['block']['create_by'] = "WMAgent"
self.data['block']['creation_date'] = int(time.time())
self.data['block']['block_size'] = 0
self.data['block']['file_count'] = 0
return
def getRequestInfo():
requestName = sys.argv[1]
print "Getting information from cmsweb about %s... %s" % (
requestName, time.strftime('%H:%M:%S'))
requestor = JSONRequests(url='https://cmsweb.cern.ch/reqmgr/reqMgr')
response = requestor.get('/request?requestName=%s' % requestName)
if response[1] != 200:
raise RuntimeError("Request information was not available!")
requestDict = response[0]
#print requestDict.get('RunWhitelist')
#print requestDict.get('RunBlacklist')
if not requestDict.get('RunWhitelist'):
requestDict['RunWhitelist'] = []
if not requestDict.get('RunBlacklist'):
requestDict['RunBlacklist'] = []
if not requestDict.get('BlockWhitelist'):
requestDict['BlockWhitelist'] = []
if not requestDict.get('BlockBlacklist'):
requestDict['BlockBlacklist'] = []
compactRequest = {
'InputDataset': requestDict['InputDataset'],
'RunWhitelist': requestDict['RunWhitelist'],
'RunBlacklist': requestDict['RunBlacklist'],
'BlockWhitelist': requestDict['BlockWhitelist'],
'BlockBlacklist': requestDict['BlockBlacklist'],
'OutputDatasets': []
}
response = requestor.get('/outputDatasetsByRequestName?requestName=%s' %
requestName)
if response[1] != 200:
raise RuntimeError("Output dataset information was not available!")
compactRequest['OutputDatasets'] = response[0]
print "Done querying cmsweb... %s" % time.strftime('%H:%M:%S')
return compactRequest
def validate_input(self, input, verb, method):
assert 'type' in input.keys(
), "no type provided - what kind of plot do you want?"
if method == 'doc':
return input
valid_data = {}
assert 'data' in input.keys() or 'url' in input.keys(
), "neither data nor url provided - please provide at least one"
if 'url' in input.keys():
match = URL_REGEX.match(input['url'])
assert match != None, "`%s' is not a valid URL" % input['url']
host = match.group(1) + '://' + match.group(3)
if match.group(4):
host += match.group(4)
uri = match.group(5)
result, status, reason, fromcache = JSONRequests(host).get(uri)
valid_data.update(result)
if 'data' in input.keys():
valid_data.update(json.loads(urllib.unquote(input['data'])))
return {'data': valid_data, 'type': input['type']}
def setUp(self):
RESTBaseUnitTest.setUp(self)
self.testInit.setupCouch("%s" % self.couchDBName, "GroupUser", "ConfigCache")
self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats")
# logging stuff from TestInit is broken, setting myself
l = logging.getLogger()
l.setLevel(logging.DEBUG)
self.params = {}
self.params['endpoint'] = self.config.getServerUrl()
self.reqService = RequestManagerDS(self.params)
self.jsonSender = JSONRequests(self.config.getServerUrl())
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
try:
r = self.jsonSender.put('request/' + schema['RequestName'], schema)
except Exception, ex:
print "Exception during set up, reason: %s" % ex
return
def __call__(self, filesetToProcess):
"""
The algorithm itself
"""
# Get configuration
initObj = WMInit()
initObj.setLogging()
initObj.setDatabaseConnection(os.getenv("DATABASE"), \
os.getenv('DIALECT'), os.getenv("DBSOCK"))
myThread = threading.currentThread()
daofactory = DAOFactory(package = "WMCore.WMBS" , \
logger = myThread.logger, \
dbinterface = myThread.dbi)
lastFileset = daofactory(classname="Fileset.ListFilesetByTask")
lastWorkflow = daofactory(classname="Workflow.LoadFromTask")
subsRun = daofactory(\
classname = "Subscriptions.LoadFromFilesetWorkflow")
successJob = daofactory(classname="Subscriptions.SucceededJobs")
allJob = daofactory(classname="Subscriptions.Jobs")
fileInFileset = daofactory(classname="Files.InFileset")
# Get the start Run if asked
startRun = (filesetToProcess.name).split(":")[3]
logging.debug("the T0Feeder is processing %s" % \
filesetToProcess.name)
logging.debug("the fileset name %s" % \
(filesetToProcess.name).split(":")[0])
fileType = (filesetToProcess.name).split(":")[2]
crabTask = filesetToProcess.name.split(":")[0]
LASTIME = filesetToProcess.lastUpdate
tries = 1
while True:
try:
myRequester = JSONRequests(url="vocms52.cern.ch:8889")
requestResult = myRequester.get("/tier0/runs")
except:
logging.debug("T0Reader call error...")
if tries == self.maxRetries:
return
else:
tries += 1
continue
logging.debug("T0ASTRunChain feeder queries done ...")
now = time.time()
break
for listRun in requestResult[0]:
if startRun != 'None' and int(listRun['run']) >= int(startRun):
if listRun['status'] =='CloseOutExport' or listRun\
['status']=='Complete' or listRun['status']=='CloseOutT1Skimming':
crabWorkflow = lastWorkflow.execute(task=crabTask)
crabFileset = lastFileset.execute\
(task=crabTask)
crabrunFileset = Fileset(\
name = crabFileset[0]["name"].split(':')[0].split\
('-Run')[0]+ '-Run' + str(listRun['run']) + ":" + \
":".join(crabFileset[0]['name'].split(':')[1:]) )
if crabrunFileset.exists() > 0:
crabrunFileset.load()
currSubs = subsRun.execute\
(crabrunFileset.id, crabWorkflow[0]['id'])
if currSubs:
listsuccessJob = successJob.execute(\
subscription=currSubs['id'])
listallJob = allJob.execute(\
subscription=currSubs['id'])
if len(listsuccessJob) == len(listallJob):
for currid in listsuccessJob:
currjob = Job(id=currid)
currjob.load()
logging.debug("Reading FJR %s" %
currjob['fwjr_path'])
jobReport = readJobReport(
currjob['fwjr_path'])
if len(jobReport) > 0:
if jobReport[0].files:
for newFile in jobReport[0].files:
logging.debug(\
"Output path %s" %newFile['LFN'])
newFileToAdd = File(\
lfn=newFile['LFN'], locations ='caf.cern.ch')
LOCK.acquire()
if newFileToAdd.exists\
() == False :
newFileToAdd.create()
else:
newFileToAdd.loadData()
LOCK.release()
listFile = \
fileInFileset.execute(filesetToProcess.id)
if {'fileid': \
newFileToAdd['id']} not in listFile:
filesetToProcess.addFile(\
newFileToAdd)
filesetToProcess\
.setLastUpdate(now)
filesetToProcess.commit()
logging.debug(\
"new file created/loaded and added by T0ASTRunChain...")
elif jobReport[0].analysisFiles:
for newFile in jobReport\
[0].analysisFiles:
logging.debug(\
"Ouput path %s " %newFile['LFN'])
newFileToAdd = File(\
lfn=newFile['LFN'], locations ='caf.cern.ch')
LOCK.acquire()
if newFileToAdd.exists\
() == False :
newFileToAdd.create()
else:
newFileToAdd.loadData()
LOCK.release()
listFile = \
fileInFileset.execute(filesetToProcess.id)
if {'fileid': newFileToAdd\
['id']} not in listFile:
logging.debug\
("%s loaded and added by T0ASTRunChain" %newFile['LFN'])
filesetToProcess.addFile\
(newFileToAdd)
filesetToProcess.\
setLastUpdate(now)
filesetToProcess.commit()
logging.debug(\
"new file created/loaded added by T0ASTRunChain...")
else:
break #Missed fjr - Try next time
# Commit the fileset
logging.debug("Test purge in T0ASTRunChain ...")
filesetToProcess.load()
LASTIME = filesetToProcess.lastUpdate
# For re-opned fileset or empty, try until the purge time
if (int(now) / 3600 - LASTIME / 3600) > self.reopenTime:
filesetToProcess.setLastUpdate(time.time())
filesetToProcess.commit()
if (int(now) / 3600 - LASTIME / 3600) > self.purgeTime:
filesetToProcess.markOpen(False)
logging.debug("Purge Done...")
def __init__(self, slaveClassName, totalSlaves, componentDir,
config, namespace = 'WMComponent', inPort = '5555',
outPort = '5558'):
"""
__init__
Constructor for the process pool. The slave class name must be based
inside the WMComponent namespace. For examples, the JobAccountant would
pass in 'JobAccountant.AccountantWorker' to run the AccountantWorker
class. All log files will be stored in the component directory that is
passed in. Each slave will have its own log file.
Note that the config is only used to determine database connection
parameters. It is not passed to the slave class. The slaveInit
parameter will be serialized and passed to the slave class's
constructor.
"""
self.enqueueIndex = 0
self.dequeueIndex = 0
self.runningWork = 0
#Use the Services.Requests JSONizer, which handles __to_json__ calls
self.jsonHandler = JSONRequests()
# heartbeat should be registered at this point
if getattr(config.Agent, "useHeartbeat", True):
self.heartbeatAPI = HeartbeatAPI(getattr(config.Agent, "componentName", "ProcPoolSlave"))
self.slaveClassName = slaveClassName
self.componentDir = componentDir
self.config = config
# Grab the python version from the current version
# Assume naming convention pythonA.B, i.e., python2.4 for v2.4.X
majorVersion = sys.version_info[0]
minorVersion = sys.version_info[1]
if majorVersion and minorVersion:
self.versionString = "python%i.%i" % (majorVersion, minorVersion)
else:
self.versionString = "python2.6"
self.workers = []
self.nSlaves = totalSlaves
self.namespace = namespace
self.inPort = inPort
self.outPort = outPort
# Pickle the config
self.configPath = os.path.join(componentDir, '%s_config.pkl' % slaveClassName)
if os.path.exists(self.configPath):
# Then we note it and overwrite it
msg = "Something's in the way of the ProcessPool config: %s" % self.configPath
logging.error(msg)
f = open(self.configPath, 'w')
cPickle.dump(config, f)
f.close()
# Set up ZMQ
try:
context = zmq.Context()
self.sender = context.socket(zmq.PUSH)
self.sender.bind("tcp://*:%s" % inPort)
self.sink = context.socket(zmq.PULL)
self.sink.bind("tcp://*:%s" % outPort)
except zmq.ZMQError:
# Try this again in a moment to see
# if it's just being held by something pre-existing
import time
time.sleep(1)
logging.error("Blocked socket on startup: Attempting sleep to give it time to clear.")
try:
context = zmq.Context()
self.sender = context.socket(zmq.PUSH)
self.sender.bind("tcp://*:%s" % inPort)
self.sink = context.socket(zmq.PULL)
self.sink.bind("tcp://*:%s" % outPort)
except Exception as ex:
msg = "Error attempting to open TCP sockets\n"
msg += str(ex)
logging.error(msg)
import traceback
print traceback.format_exc()
raise ProcessPoolException(msg)
# Now actually create the slaves
self.createSlaves()
return
if not os.path.exists(configPath):
# We can do nothing -
logging.error("Something in the way of the config path")
sys.exit(1)
f = open(configPath, 'r')
config = cPickle.load(f)
f.close()
# Setup DB
wmInit = WMInit()
setupDB(config, wmInit)
# Create JSON handler
jsonHandler = JSONRequests()
wmFactory = WMFactory(name = "slaveFactory", namespace = namespace)
slaveClass = wmFactory.loadObject(classname = slaveClassName, args = config)
logging.info("Have slave class")
while(True):
encodedInput = receiver.recv()
try:
input = jsonHandler.decode(encodedInput)
except Exception as ex:
logging.error("Error decoding: %s" % str(ex))
break
def testGoodGetJSON(self):
headers={"Accept": "application/json"}
json = JSONRequests('localhost:8010')
self.runReq(self.goodurls, json, 200)
filesetToProcess.commit()
LOCK.release()
else:
logging.debug("nothing to do...")
# For re-opned fileset or empty, try until the purge time
if (int(now) / 3600 - LASTIME / 3600) > self.reopenTime:
filesetToProcess.setLastUpdate(time.time())
filesetToProcess.commit()
if LASTIME:
myRequester = JSONRequests(url="vocms52.cern.ch:8889")
requestResult = myRequester.get("/tier0/runs")
for listRun in requestResult[0]:
if int(startRun) <= int(listRun['run']):
if listRun['status'] =='CloseOutExport' or \
listRun['status'] =='Complete' or listRun['status'] ==\
'CloseOutT1Skimming':
closeFileset = Fileset( name = (((\
filesetToProcess.name).split(':')[0]).split('/')[0])+'/'+\
(((filesetToProcess.name).split(':')[0]).split('/')[1]\
)+'/'+(((filesetToProcess.name).split(':')[0]).split('/')\
[2])+'/'+((((filesetToProcess.name).split(':')[0]).split\
def __call__(self, filesetToProcess):
"""
The algorithm itself
"""
global LOCK
# Get configuration
initObj = WMInit()
initObj.setLogging()
initObj.setDatabaseConnection(os.getenv("DATABASE"), \
os.getenv('DIALECT'), os.getenv("DBSOCK"))
myThread = threading.currentThread()
daofactory = DAOFactory(package = "WMCore.WMBS" , \
logger = myThread.logger, \
dbinterface = myThread.dbi)
locationNew = daofactory(classname="Locations.New")
getFileLoc = daofactory(classname="Files.GetLocation")
fileInFileset = daofactory(classname="Files.InFileset")
logging.debug("the T0Feeder is processing %s" % \
filesetToProcess.name)
logging.debug("the fileset name %s" % \
(filesetToProcess.name).split(":")[0])
# Get the start Run if asked
startRun = (filesetToProcess.name).split(":")[3]
fileType = (filesetToProcess.name).split(":")[2]
LASTIME = filesetToProcess.lastUpdate
# url builder
primaryDataset = ((filesetToProcess.name).split(":")[0]).split('/')[1]
processedDataset = ((
filesetToProcess.name).split(":")[0]).split('/')[2]
dataTier = ((filesetToProcess.name\
).split(":")[0]).split('/')[3]
url = "/tier0/listfilesoverinterval/%s/%s/%s/%s/%s" % \
(fileType, LASTIME, primaryDataset,processedDataset, dataTier)
tries = 1
while True:
try:
myRequester = JSONRequests(url="vocms52.cern.ch:8889")
requestResult = myRequester.get(\
url+"/"+"?return_type=text/json%2Bdas")
newFilesList = requestResult[0]["results"]
except:
logging.debug("T0Reader call error...")
if tries == self.maxRetries:
return
else:
tries += 1
continue
logging.debug("T0 queries done ...")
now = time.time()
LASTIME = int(newFilesList['end_time']) + 1
break
# process all files
if len(newFilesList['files']):
try:
locationNew.execute(siteName="caf.cern.ch",
seName="caf.cern.ch")
except Exception as e:
logging.debug("Error when adding new location...")
logging.debug(e)
logging.debug(format_exc())
for files in newFilesList['files']:
# Assume parents aren't asked
newfile = File(str(files['lfn']), \
size = files['file_size'], events = files['events'])
try:
LOCK.acquire()
if newfile.exists() == False:
newfile.create()
for run in files['runs']:
runSet = set()
runSet.add(Run(run, *files['runs'][run]))
newfile.addRunSet(runSet)
else:
newfile.loadData()
fileLoc = getFileLoc.execute(file=files['lfn'])
if 'caf.cern.ch' not in fileLoc:
newfile.setLocation("caf.cern.ch")
# else:
# logging.debug("File already associated to %s" %fileLoc)
LOCK.release()
if len(newfile["runs"]):
val = 0
for run in newfile['runs']:
if run.run < int(startRun):
val = 1
break
if not val:
listFile = fileInFileset.execute\
(filesetToProcess.id)
if {'fileid': newfile['id']} not in listFile:
filesetToProcess.addFile(newfile)
filesetToProcess.setLastUpdate\
(int(newFilesList['end_time']) + 1)
filesetToProcess.commit()
logging.debug(
"new file created/loaded added by T0AST..."
)
except Exception as e:
logging.debug("Error when adding new files in T0AST...")
logging.debug(e)
logging.debug(format_exc())
LOCK.release()
filesetToProcess.commit()
else:
logging.debug("nothing to do in T0AST...")
# For reopned fileset or empty
# try until the purge time is reached
if (int(now) / 3600 - LASTIME / 3600) > self.reopenTime:
filesetToProcess.setLastUpdate(time.time())
filesetToProcess.commit()
# Commit the fileset
logging.debug("Test purge in T0AST ...")
filesetToProcess.load()
LASTIME = filesetToProcess.lastUpdate
if (int(now) / 3600 - LASTIME / 3600) > self.purgeTime:
filesetToProcess.markOpen(False)
logging.debug("Purge Done...")
filesetToProcess.commit()
def testBaadGetJSON(self):
headers = {"Accept": "application/json"}
json = JSONRequests('localhost:8308')
self.runReq(self.badurls, json, 400)
def __call__(self, filesetToProcess):
"""
The algorithm itself
"""
global LOCK
# Get configuration
initObj = WMInit()
initObj.setLogging()
initObj.setDatabaseConnection(os.getenv("DATABASE"), \
os.getenv('DIALECT'), os.getenv("DBSOCK"))
myThread = threading.currentThread()
daofactory = DAOFactory(package = "WMCore.WMBS" , \
logger = myThread.logger, \
dbinterface = myThread.dbi)
locationNew = daofactory(classname="Locations.New")
getFileLoc = daofactory(classname="Files.GetLocation")
logging.debug("the T0Feeder is processing %s" % \
filesetToProcess.name)
logging.debug("the fileset name %s" % \
(filesetToProcess.name).split(":")[0])
startRun = (filesetToProcess.name).split(":")[3]
fileType = (filesetToProcess.name).split(":")[2]
# url builder
primaryDataset = ((filesetToProcess.name).split(":")[0]).split('/')[1]
processedDataset = ((
filesetToProcess.name).split(":")[0]).split('/')[2]
dataTier = (((filesetToProcess.name\
).split(":")[0]).split('/')[3]).split('-')[0]
# Fisrt call to T0 db for this fileset
# Here add test for the closed fileset
LASTIME = filesetToProcess.lastUpdate
url = "/tier0/listfilesoverinterval/%s/%s/%s/%s/%s" % \
(fileType, LASTIME, primaryDataset,processedDataset, dataTier)
tries = 1
while True:
try:
myRequester = JSONRequests(url="vocms52.cern.ch:8889")
requestResult = myRequester.get(\
url+"/"+"?return_type=text/json%2Bdas")
newFilesList = requestResult[0]["results"]
except:
logging.debug("T0Reader call error...")
if tries == self.maxRetries:
return
else:
tries += 1
continue
logging.debug("T0ASTRun queries done ...")
now = time.time()
filesetToProcess.last_update = now
LASTIME = int(newFilesList['end_time']) + 1
break
# process all files
if len(newFilesList['files']):
LOCK.acquire()
try:
locationNew.execute(siteName="caf.cern.ch",
seName="caf.cern.ch")
except Exception as e:
logging.debug("Error when adding new location...")
logging.debug(e)
logging.debug(format_exc())
for files in newFilesList['files']:
# Assume parents aren't asked
newfile = File(str(files['lfn']), \
size = files['file_size'], events = files['events'])
try:
if newfile.exists() == False:
newfile.create()
else:
newfile.loadData()
#Add run test if already exist
for run in files['runs']:
if startRun != 'None' and int(startRun) <= int(run):
# ToDo: Distinguish between
# filestA-RunX and filesetA-Run[0-9]*
filesetRun = Fileset( name = (((\
filesetToProcess.name).split(':')[0]).split('/')[0]\
)+'/'+(((filesetToProcess.name).split(':')[0]).split\
('/')[1])+'/'+(((filesetToProcess.name).split(':')[0]\
).split('/')[2])+'/'+((((filesetToProcess.name).split\
(':')[0]).split('/')[3]).split('-')[0])+'-'+'Run'+str\
(run)+":"+":".join((filesetToProcess.name).split(':')[1:] \
) )
if filesetRun.exists() == False:
filesetRun.create()
else:
filesetRun.loadData()
# Add test runs already there
# (for growing dataset) -
# to support file with different runs and lumi
if not newfile['runs']:
runSet = set()
runSet.add(Run(run, *files['runs'][run]))
newfile.addRunSet(runSet)
fileLoc = getFileLoc.execute(file=files['lfn'])
if 'caf.cern.ch' not in fileLoc:
newfile.setLocation("caf.cern.ch")
filesetRun.addFile(newfile)
logging.debug(
"new file created/loaded added by T0ASTRun...")
filesetRun.commit()
except Exception as e:
logging.debug("Error when adding new files in T0ASTRun...")
logging.debug(e)
logging.debug(format_exc())
filesetToProcess.setLastUpdate\
(int(newFilesList['end_time']) + 1)
filesetToProcess.commit()
LOCK.release()
else:
logging.debug("nothing to do...")
# For re-opned fileset or empty, try until the purge time
if (int(now) / 3600 - LASTIME / 3600) > self.reopenTime:
filesetToProcess.setLastUpdate(time.time())
filesetToProcess.commit()
if LASTIME:
myRequester = JSONRequests(url="vocms52.cern.ch:8889")
requestResult = myRequester.get("/tier0/runs")
for listRun in requestResult[0]:
if int(startRun) <= int(listRun['run']):
if listRun['status'] =='CloseOutExport' or \
listRun['status'] =='Complete' or listRun['status'] ==\
'CloseOutT1Skimming':
closeFileset = Fileset( name = (((\
filesetToProcess.name).split(':')[0]).split('/')[0])+'/'+\
(((filesetToProcess.name).split(':')[0]).split('/')[1]\
)+'/'+(((filesetToProcess.name).split(':')[0]).split('/')\
[2])+'/'+((((filesetToProcess.name).split(':')[0]).split\
('/')[3]).split('-')[0])+'-'+'Run'+str(listRun['run'])\
+":"+":".join((filesetToProcess.name).split(':')[1:] ) )
if closeFileset.exists() != False:
closeFileset = Fileset(id=closeFileset.exists())
closeFileset.loadData()
if closeFileset.open == True:
closeFileset.markOpen(False)
# Commit the fileset
filesetToProcess.commit()
# Commit the fileset
logging.debug("Test purge in T0ASTRun ...")
filesetToProcess.load()
LASTIME = filesetToProcess.lastUpdate
if (int(now) / 3600 - LASTIME / 3600) > self.purgeTime:
filesetToProcess.markOpen(False)
logging.debug("Purge Done...")
filesetToProcess.commit()
def testSpecialCharacterPasswords(self):
url = 'http://*****:*****@ssw:rd@localhost:6666'
req = JSONRequests(url)
self.assertEqual(req['host'], 'http://localhost:6666')
self.assertEqual(req.additionalHeaders['Authorization'], 'Basic dXNlcm5hbWU6cEBzc3c6cmQ=')
def __init__(self, reqMgrUrl):
self.reqMgrUrl = reqMgrUrl
self.restSender = JSONRequests(reqMgrUrl)
d = dict(endpoint=self.reqMgrUrl)
self.reqMgrService = RequestManager(d)