class UserFileCacheTest(unittest.TestCase): """ Unit tests for UserFileCache Service """ def testChecksum(self): """ Tests checksum method """ self.ufc = UserFileCache() checksum1 = self.ufc.checksum(fileName=path.join(getTestBase(), '../data/ewv_crab_EwvAnalysis_31_111229_140959_publish.tgz')) checksum2 = self.ufc.checksum(fileName=path.join(getTestBase(), '../data/ewv_crab_EwvAnalysis_31_resubmit_111229_144319_publish.tgz')) self.assertTrue(checksum1) self.assertTrue(checksum2) self.assertFalse(checksum1 == checksum2) self.assertRaises(IOError, self.ufc.checksum, **{'fileName': 'does_not_exist'}) return def testUploadDownload(self): if 'UFCURL' in os.environ: currdir = getTestBase() upfile = path.join(currdir, 'WMCore_t/Services_t/UserFileCache_t/test_file.tgz') #file to upload ufc = UserFileCache({'endpoint':os.environ['UFCURL']}) #named upload/download res = ufc.upload(upfile, 'name_publish.tgz') ufc.download(name=res['name'], output='name_publish.tgz') #hashkey upload/download res = ufc.upload(upfile) ufc.download(res['hashkey'], output='pippo_publish_down.tgz')
class UserFileCacheTest(unittest.TestCase): """ Unit tests for UserFileCache Service """ def testChecksum(self): """ Tests checksum method """ self.ufc = UserFileCache() checksum1 = self.ufc.checksum(fileName=path.join(getTestBase(), 'WMCore_t/Services_t/UserFileCache_t/ewv_crab_EwvAnalysis_31_111229_140959_publish.tgz')) checksum2 = self.ufc.checksum(fileName=path.join(getTestBase(), 'WMCore_t/Services_t/UserFileCache_t/ewv_crab_EwvAnalysis_31_resubmit_111229_144319_publish.tgz')) self.assertTrue(checksum1) self.assertTrue(checksum2) self.assertFalse(checksum1 == checksum2) self.assertRaises(IOError, self.ufc.checksum, **{'fileName': 'does_not_exist'}) return def testUploadDownload(self): if 'UFCURL' in os.environ: currdir = getTestBase() upfile = path.join(currdir, 'WMCore_t/Services_t/UserFileCache_t/test_file.tgz') #file to upload ufc = UserFileCache({'endpoint':os.environ['UFCURL']}) #named upload/download res = ufc.upload(upfile, 'name_publish.tgz') ufc.download(name=res['name'], output='name_publish.tgz') #hashkey upload/download res = ufc.upload(upfile) ufc.download(res['hashkey'], output='pippo_publish_down.tgz')
def __call__(self): self.logger.info('Getting the tarball hash key') tarballdir = glob.glob(self.requestarea+'/inputs/*.tgz') if len(tarballdir) != 1: self.logger.info('%sError%s: Could not find tarball or there is more than one tarball'% (colors.RED, colors.NORMAL)) raise ConfigurationException tarballdir = tarballdir[0] #checking task status self.logger.info('Checking task status') serverFactory = CRABClient.Emulator.getEmulator('rest') server = serverFactory(self.serverurl, self.proxyfilename, self.proxyfilename, version=__version__) dictresult, status, _ = server.get(self.uri, data = {'workflow': self.cachedinfo['RequestName'], 'verbose': 0}) dictresult = dictresult['result'][0] #take just the significant part if status != 200: msg = "Problem retrieving task status:\ninput: %s\noutput: %s\nreason: %s" % (str(self.cachedinfo['RequestName']), str(dictresult), str(reason)) raise RESTCommunicationException(msg) self.logger.info('Task status: %s' % dictresult['status']) accepstate = ['KILLED','FINISHED','FAILED','KILLFAILED', 'COMPLETED'] if dictresult['status'] not in accepstate: msg = ('%sError%s: Only tasks with these status can be purged: {0}'.format(accepstate) % (colors.RED, colors.NORMAL)) raise ConfigurationException(msg) #getting the cache url cacheresult = {} scheddresult = {} gsisshdict = {} if not self.options.scheddonly: baseurl = getUrl(self.instance, resource='info') cacheurl = server_info('backendurls', self.serverurl, self.proxyfilename, baseurl) cacheurl = cacheurl['cacheSSL'] cacheurldict = {'endpoint': cacheurl, 'pycurl': True} ufc = UserFileCache(cacheurldict) hashkey = ufc.checksum(tarballdir) self.logger.info('Tarball hashkey: %s' %hashkey) self.logger.info('Attempting to remove task file from crab server cache') try: ufcresult = ufc.removeFile(hashkey) except HTTPException, re: if re.headers.has_key('X-Error-Info') and 'Not such file' in re.headers['X-Error-Info']: self.logger.info('%sError%s: Failed to find task file in crab server cache; the file might have been already purged' % (colors.RED,colors.NORMAL)) raise HTTPException , re if ufcresult == '': self.logger.info('%sSuccess%s: Successfully removed task files from crab server cache' % (colors.GREEN, colors.NORMAL)) cacheresult = 'SUCCESS' else: self.logger.info('%sError%s: Failed to remove task files from crab server cache' % (colors.RED, colors.NORMAL)) cacheresult = 'FAILED'
class UserFileCacheTest(unittest.TestCase): """ Unit tests for UserFileCache Service """ def setUp(self): """ Setup for unit tests """ self.ufc = UserFileCache() def testChecksum(self): """ Tests checksum method """ checksum1 = self.ufc.checksum(fileName=path.join(getTestBase(), '../data/ewv_crab_EwvAnalysis_31_111229_140959_publish.tgz')) checksum2 = self.ufc.checksum(fileName=path.join(getTestBase(), '../data/ewv_crab_EwvAnalysis_31_resubmit_111229_144319_publish.tgz')) self.assertTrue(checksum1) self.assertTrue(checksum2) self.assertFalse(checksum1 == checksum2) self.assertRaises(IOError, self.ufc.checksum, **{'fileName': 'does_not_exist'}) return
class UserFileCacheTest(unittest.TestCase): """ Unit tests for UserFileCache Service """ def testChecksum(self): """ Tests checksum method """ self.ufc = UserFileCache() checksum1 = self.ufc.checksum( fileName=path.join(getTestBase(), "../data/ewv_crab_EwvAnalysis_31_111229_140959_publish.tgz") ) checksum2 = self.ufc.checksum( fileName=path.join(getTestBase(), "../data/ewv_crab_EwvAnalysis_31_resubmit_111229_144319_publish.tgz") ) self.assertTrue(checksum1) self.assertTrue(checksum2) self.assertFalse(checksum1 == checksum2) self.assertRaises(IOError, self.ufc.checksum, **{"fileName": "does_not_exist"}) return def testUploadDownload(self): if "UFCURL" in os.environ: currdir = getTestBase() upfile = path.join(currdir, "WMCore_t/Services_t/UserFileCache_t/test_file.tgz") # file to upload ufc = UserFileCache({"endpoint": os.environ["UFCURL"]}) # named upload/download res = ufc.upload(upfile, "name_publish.tgz") ufc.download(name=res["name"], output="name_publish.tgz") # hashkey upload/download res = ufc.upload(upfile) ufc.download(res["hashkey"], output="pippo_publish_down.tgz")
def __call__(self): self.logger.info('Getting the tarball hash key') tarballdir=glob.glob(self.requestarea+'/inputs/*.tgz') if len(tarballdir) != 1 : self.logger.info('%sError%s: Could not find tarball or there is more than one tarball'% (colors.RED, colors.NORMAL)) raise ConfigurationException tarballdir=tarballdir[0] #checking task status self.logger.info('Checking task status') server = HTTPRequests(self.serverurl, self.proxyfilename, self.proxyfilename, version=__version__) dictresult, status, _ = server.get(self.uri, data = { 'workflow' : self.cachedinfo['RequestName'], 'verbose': 0 }) dictresult = dictresult['result'][0] #take just the significant part if status != 200: msg = "Problem retrieving status:\ninput:%s\noutput:%s\nreason:%s" % (str(self.cachedinfo['RequestName']), str(dictresult), str(reason)) raise RESTCommunicationException(msg) self.logger.info('Task status: %s' % dictresult['status']) accepstate = ['KILLED','FINISHED','FAILED','KILLFAILED', 'COMPLETED'] if dictresult['status'] not in accepstate: msg = ('%sError%s: Only task with this status can be purge: {0}'.format(accepstate) % (colors.RED, colors.NORMAL)) raise ConfigurationException(msg) #getting the cache url if not self.options.scheddonly: baseurl=self.getUrl(self.instance, resource='info') cacheurl=server_info('backendurls', self.serverurl, self.proxyfilename, baseurl) cacheurl=cacheurl['cacheSSL'] cacheurldict={'endpoint' : cacheurl, 'pycurl': True} ufc = UserFileCache(cacheurldict) hashkey = ufc.checksum(tarballdir) self.logger.info('Tarball hashkey :%s' %hashkey) self.logger.info('Attempting to clean user file cache') ufcresult = ufc.removeFile(hashkey) if ufcresult == '' : self.logger.info('%sSuccess%s: Successfully remove file from cache' % (colors.GREEN, colors.NORMAL)) else: self.logger.info('%sError%s: Failed to remove the file from cache' % (colors.RED, colors.NORMAL)) if not self.options.cacheonly: self.logger.info('Getting the schedd address') baseurl=self.getUrl(self.instance, resource='info') try: sceddaddress = server_info('scheddaddress', self.serverurl, self.proxyfilename, baseurl, workflow = self.cachedinfo['RequestName'] ) except HTTPException, he: self.logger.info('%sError%s: Failed to get the schedd address' % (colors.RED, colors.NORMAL)) raise HTTPException,he self.logger.debug('%sSuccess%s: Successfully getting schedd address' % (colors.GREEN, colors.NORMAL)) self.logger.debug('Schedd address: %s' % sceddaddress) self.logger.info('Attempting to clean user file schedd') gssishrm = 'gsissh -o ConnectTimeout=60 -o PasswordAuthentication=no ' + sceddaddress + ' rm -rf ' + self.cachedinfo['RequestName'] self.logger.debug('gsissh command: %s' % gssishrm) delprocess=subprocess.Popen(gssishrm, stdout= subprocess.PIPE, stderr= subprocess.PIPE, shell=True) stdout, stderr = delprocess.communicate() exitcode = delprocess.returncode if exitcode == 0 : self.logger.info('%sSuccess%s: Successfully remove task from scehdd' % (colors.GREEN, colors.NORMAL)) else : self.logger.info('%sError%s: Failed to remove task from schedd' % (colors.RED, colors.NORMAL)) self.logger.debug('gsissh stdout: %s\ngsissh stderr: %s\ngsissh exitcode: %s' % (stdout,stderr,exitcode))
def __call__(self): self.logger.info('Getting the tarball hash key') tarballdir = glob.glob(self.requestarea + '/inputs/*.tgz') if len(tarballdir) != 1: self.logger.info( '%sError%s: Could not find tarball or there is more than one tarball' % (colors.RED, colors.NORMAL)) raise ConfigurationException tarballdir = tarballdir[0] #checking task status self.logger.info('Checking task status') serverFactory = CRABClient.Emulator.getEmulator('rest') server = serverFactory(self.serverurl, self.proxyfilename, self.proxyfilename, version=__version__) dictresult, status, _ = server.get(self.uri, data={ 'workflow': self.cachedinfo['RequestName'], 'verbose': 0 }) dictresult = dictresult['result'][0] #take just the significant part if status != 200: msg = "Problem retrieving task status:\ninput: %s\noutput: %s\nreason: %s" % ( str(self.cachedinfo['RequestName']), str(dictresult), str(reason)) raise RESTCommunicationException(msg) self.logger.info('Task status: %s' % dictresult['status']) accepstate = [ 'KILLED', 'FINISHED', 'FAILED', 'KILLFAILED', 'COMPLETED' ] if dictresult['status'] not in accepstate: msg = ('%sError%s: Only tasks with these status can be purged: {0}' .format(accepstate) % (colors.RED, colors.NORMAL)) raise ConfigurationException(msg) #getting the cache url cacheresult = {} scheddresult = {} gsisshdict = {} if not self.options.scheddonly: baseurl = getUrl(self.instance, resource='info') cacheurl = server_info('backendurls', self.serverurl, self.proxyfilename, baseurl) cacheurl = cacheurl['cacheSSL'] cacheurldict = {'endpoint': cacheurl, 'pycurl': True} ufc = UserFileCache(cacheurldict) hashkey = ufc.checksum(tarballdir) self.logger.info('Tarball hashkey: %s' % hashkey) self.logger.info( 'Attempting to remove task file from crab server cache') try: ufcresult = ufc.removeFile(hashkey) except HTTPException, re: if re.headers.has_key( 'X-Error-Info' ) and 'Not such file' in re.headers['X-Error-Info']: self.logger.info( '%sError%s: Failed to find task file in crab server cache; the file might have been already purged' % (colors.RED, colors.NORMAL)) raise HTTPException, re if ufcresult == '': self.logger.info( '%sSuccess%s: Successfully removed task files from crab server cache' % (colors.GREEN, colors.NORMAL)) cacheresult = 'SUCCESS' else: self.logger.info( '%sError%s: Failed to remove task files from crab server cache' % (colors.RED, colors.NORMAL)) cacheresult = 'FAILED'