def getRequest(operation): """ fake requestDict :param str operation: sub-request operation attribute """ requestContainer = RequestContainer(init=False) requestContainer.setJobID(1) #requestContainer.setOwnerDN( "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba" ) requestContainer.setOwnerGroup("lhcb_user") requestContainer.setDIRACSetup("LHCb-Production") requestContainer.setSourceComponent(None) requestContainer.setCreationTime("0000-00-00 00:00:00") requestContainer.setLastUpdate("2011-12-01 04:57:02") requestContainer.setStatus("Waiting") requestContainer.setAttribute("RequestID", 123456789) requestContainer.initiateSubRequest("transfer") subRequestDict = { "Status": "Waiting", "SubRequestID": 2222222, "Operation": operation, "Arguments": None, "ExecutionOrder": 0, "SourceSE": None, "TargetSE": "CERN-USER", "Catalogue": "LcgFileCatalogCombined", "CreationTime": "2011-12-01 04:57:02", "SubmissionTime": "2011-12-01 04:57:02", "LastUpdate": "2011-12-01 20:14:22" } requestContainer.setSubRequestAttributes(0, "transfer", subRequestDict) files = [{ "FileID": 3333333, "LFN": "/lhcb/user/c/cibak/11889/11889410/test.zzz", "Size": 44444444, "PFN": "srm://srm-lhcb.gridpp.rl.ac.uk/castor/ads.rl.ac.uk/prod/lhcb/user/c/cibak/11889/11889410/test.zzz", "GUID": "5P13RD4L-4J5L-3D21-U5P1-3RD4L4J5P13R", "Md5": None, "Addler": "92b85e26", "Attempt": 1, "Status": "Waiting" }] requestContainer.setSubRequestFiles(0, "transfer", files) return { "OK": True, "Value": { "requestName": "%s.xml" % operation, "requestString": requestContainer.toXML_new()["Value"], "requestObj": requestContainer, "jobID": 1, "executionOrder": 0, "sourceServer": "foobarserver" } }
def sweeper(cls): """ move cached request to the central request manager :param self: self reference """ cacheDir = cls.cacheDir() ## cache dir empty? if not os.listdir(cacheDir): gLogger.always("sweeper: CacheDir %s is empty, nothing to do" % cacheDir) return S_OK() else: ## read 10 cache dir files, the oldest first cachedRequests = [ os.path.abspath(requestFile) for requestFile in sorted(filter(os.path.isfile, [ os.path.join(cacheDir, requestName) for requestName in os.listdir(cacheDir) ]), key=os.path.getctime) ][:30] ## set cached requests to the central RequestManager for cachedFile in cachedRequests: try: requestString = "".join(open(cachedFile, "r").readlines()) cachedRequest = RequestContainer(requestString) requestName = cachedRequest.getAttribute( "RequestName")["Value"] ## cibak: hack for DISET requests if requestName == "Unknown": cachedRequest.setAttribute("RequestName", makeGuid()) requestName = cachedRequest.getAttribute( "RequestName")["Value"] setRequest = cls.requestManager().setRequest( requestName, requestString) if not setRequest["OK"]: gLogger.error( "sweeper: unable to set request '%s' @ RequestManager: %s" % (requestName, setRequest["Message"])) continue gLogger.info( "sweeper: successfully set request '%s' @ RequestManager" % requestName) os.unlink(cachedFile) except Exception, error: gLogger.exception("sweeper: hit by exception %s" % str(error)) return S_ERROR("sweeper: hit by exception: %s" % str(error)) return S_OK()
def getRequest(operation): """ fake requestDict :param str operation: sub-request operation attribute """ requestContainer = RequestContainer(init=False) requestContainer.setJobID(1) requestContainer.setOwnerDN("/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba") requestContainer.setOwnerGroup("lhcb_user") requestContainer.setDIRACSetup("LHCb-Production") requestContainer.setSourceComponent(None) requestContainer.setCreationTime("0000-00-00 00:00:00") requestContainer.setLastUpdate("2011-12-01 04:57:02") requestContainer.setStatus("Waiting") requestContainer.setAttribute("RequestID", 123456789) requestContainer.initiateSubRequest("transfer") subRequestDict = { "Status": "Waiting", "SubRequestID": 2222222, "Operation": operation, "Arguments": None, "ExecutionOrder": 0, "SourceSE": None, "TargetSE": "CERN-USER,PIC-USER", "Catalogue": "LcgFileCatalogCombined", "CreationTime": "2011-12-01 04:57:02", "SubmissionTime": "2011-12-01 04:57:02", "LastUpdate": "2011-12-01 20:14:22", } requestContainer.setSubRequestAttributes(0, "transfer", subRequestDict) files = [ { "FileID": 3333333, "LFN": "/lhcb/user/c/cibak/11889/11889410/test.zzz", "Size": 44444444, "PFN": "srm://srm-lhcb.gridpp.rl.ac.uk/castor/ads.rl.ac.uk/prod/lhcb/user/c/cibak/11889/11889410/test.zzz", "GUID": "5P13RD4L-4J5L-3D21-U5P1-3RD4L4J5P13R", "Md5": None, "Addler": "92b85e26", "Attempt": 1, "Status": "Waiting", } ] requestContainer.setSubRequestFiles(0, "transfer", files) return requestContainer
def sweeper( cls ): """ move cached request to the central request manager :param cls: class reference """ cacheDir = cls.cacheDir() ## cache dir empty? if not os.listdir( cacheDir ): gLogger.always("sweeper: CacheDir %s is empty, nothing to do" % cacheDir ) return S_OK() else: ## read 10 cache dir files, the oldest first cachedRequests = [ os.path.abspath( requestFile ) for requestFile in sorted( filter( os.path.isfile, [ os.path.join( cacheDir, requestName ) for requestName in os.listdir( cacheDir ) ] ), key = os.path.getctime ) ][:30] ## set cached requests to the central RequestManager for cachedFile in cachedRequests: try: requestString = "".join( open( cachedFile, "r" ).readlines() ) cachedRequest = RequestContainer( requestString ) requestName = cachedRequest.getAttribute("RequestName")["Value"] ## cibak: hack for DISET requests if requestName == "Unknown": cachedRequest.setAttribute( "RequestName", makeGuid() ) requestName = cachedRequest.getAttribute("RequestName")["Value"] setRequest = cls.requestManager().setRequest( requestName, requestString ) if not setRequest["OK"]: gLogger.error("sweeper: unable to set request '%s' @ RequestManager: %s" % ( requestName, setRequest["Message"] ) ) continue gLogger.info("sweeper: successfully set request '%s' @ RequestManager" % requestName ) os.unlink( cachedFile ) except Exception, error: gLogger.exception( "sweeper: hit by exception %s" % str(error) ) return S_ERROR( "sweeper: hit by exception: %s" % str(error) ) return S_OK()