class UserFileCacheTest(unittest.TestCase):
"""
Unit tests for UserFileCache Service
"""
def testChecksum(self):
"""
Tests checksum method
"""
self.ufc = UserFileCache()
checksum1 = self.ufc.checksum(fileName=path.join(getTestBase(), '../data/ewv_crab_EwvAnalysis_31_111229_140959_publish.tgz'))
checksum2 = self.ufc.checksum(fileName=path.join(getTestBase(), '../data/ewv_crab_EwvAnalysis_31_resubmit_111229_144319_publish.tgz'))
self.assertTrue(checksum1)
self.assertTrue(checksum2)
self.assertFalse(checksum1 == checksum2)
self.assertRaises(IOError, self.ufc.checksum, **{'fileName': 'does_not_exist'})
return
def testUploadDownload(self):
if 'UFCURL' in os.environ:
currdir = getTestBase()
upfile = path.join(currdir, 'WMCore_t/Services_t/UserFileCache_t/test_file.tgz') #file to upload
ufc = UserFileCache({'endpoint':os.environ['UFCURL']})
#named upload/download
res = ufc.upload(upfile, 'name_publish.tgz')
ufc.download(name=res['name'], output='name_publish.tgz')
#hashkey upload/download
res = ufc.upload(upfile)
ufc.download(res['hashkey'], output='pippo_publish_down.tgz')
class UserFileCacheTest(unittest.TestCase):
"""
Unit tests for UserFileCache Service
"""
def testChecksum(self):
"""
Tests checksum method
"""
self.ufc = UserFileCache()
checksum1 = self.ufc.checksum(fileName=path.join(getTestBase(), 'WMCore_t/Services_t/UserFileCache_t/ewv_crab_EwvAnalysis_31_111229_140959_publish.tgz'))
checksum2 = self.ufc.checksum(fileName=path.join(getTestBase(), 'WMCore_t/Services_t/UserFileCache_t/ewv_crab_EwvAnalysis_31_resubmit_111229_144319_publish.tgz'))
self.assertTrue(checksum1)
self.assertTrue(checksum2)
self.assertFalse(checksum1 == checksum2)
self.assertRaises(IOError, self.ufc.checksum, **{'fileName': 'does_not_exist'})
return
def testUploadDownload(self):
if 'UFCURL' in os.environ:
currdir = getTestBase()
upfile = path.join(currdir, 'WMCore_t/Services_t/UserFileCache_t/test_file.tgz') #file to upload
ufc = UserFileCache({'endpoint':os.environ['UFCURL']})
#named upload/download
res = ufc.upload(upfile, 'name_publish.tgz')
ufc.download(name=res['name'], output='name_publish.tgz')
#hashkey upload/download
res = ufc.upload(upfile)
ufc.download(res['hashkey'], output='pippo_publish_down.tgz')
def __call__(self):
self.logger.info('Getting the tarball hash key')
tarballdir = glob.glob(self.requestarea+'/inputs/*.tgz')
if len(tarballdir) != 1:
self.logger.info('%sError%s: Could not find tarball or there is more than one tarball'% (colors.RED, colors.NORMAL))
raise ConfigurationException
tarballdir = tarballdir[0]
#checking task status
self.logger.info('Checking task status')
serverFactory = CRABClient.Emulator.getEmulator('rest')
server = serverFactory(self.serverurl, self.proxyfilename, self.proxyfilename, version=__version__)
dictresult, status, _ = server.get(self.uri, data = {'workflow': self.cachedinfo['RequestName'], 'verbose': 0})
dictresult = dictresult['result'][0] #take just the significant part
if status != 200:
msg = "Problem retrieving task status:\ninput: %s\noutput: %s\nreason: %s" % (str(self.cachedinfo['RequestName']), str(dictresult), str(reason))
raise RESTCommunicationException(msg)
self.logger.info('Task status: %s' % dictresult['status'])
accepstate = ['KILLED','FINISHED','FAILED','KILLFAILED', 'COMPLETED']
if dictresult['status'] not in accepstate:
msg = ('%sError%s: Only tasks with these status can be purged: {0}'.format(accepstate) % (colors.RED, colors.NORMAL))
raise ConfigurationException(msg)
#getting the cache url
cacheresult = {}
scheddresult = {}
gsisshdict = {}
if not self.options.scheddonly:
baseurl = getUrl(self.instance, resource='info')
cacheurl = server_info('backendurls', self.serverurl, self.proxyfilename, baseurl)
cacheurl = cacheurl['cacheSSL']
cacheurldict = {'endpoint': cacheurl, 'pycurl': True}
ufc = UserFileCache(cacheurldict)
hashkey = ufc.checksum(tarballdir)
self.logger.info('Tarball hashkey: %s' %hashkey)
self.logger.info('Attempting to remove task file from crab server cache')
try:
ufcresult = ufc.removeFile(hashkey)
except HTTPException, re:
if re.headers.has_key('X-Error-Info') and 'Not such file' in re.headers['X-Error-Info']:
self.logger.info('%sError%s: Failed to find task file in crab server cache; the file might have been already purged' % (colors.RED,colors.NORMAL))
raise HTTPException , re
if ufcresult == '':
self.logger.info('%sSuccess%s: Successfully removed task files from crab server cache' % (colors.GREEN, colors.NORMAL))
cacheresult = 'SUCCESS'
else:
self.logger.info('%sError%s: Failed to remove task files from crab server cache' % (colors.RED, colors.NORMAL))
cacheresult = 'FAILED'
class UserFileCacheTest(unittest.TestCase):
"""
Unit tests for UserFileCache Service
"""
def setUp(self):
"""
Setup for unit tests
"""
self.ufc = UserFileCache()
def testChecksum(self):
"""
Tests checksum method
"""
checksum1 = self.ufc.checksum(fileName=path.join(getTestBase(), '../data/ewv_crab_EwvAnalysis_31_111229_140959_publish.tgz'))
checksum2 = self.ufc.checksum(fileName=path.join(getTestBase(), '../data/ewv_crab_EwvAnalysis_31_resubmit_111229_144319_publish.tgz'))
self.assertTrue(checksum1)
self.assertTrue(checksum2)
self.assertFalse(checksum1 == checksum2)
self.assertRaises(IOError, self.ufc.checksum, **{'fileName': 'does_not_exist'})
return
class UserFileCacheTest(unittest.TestCase):
"""
Unit tests for UserFileCache Service
"""
def testChecksum(self):
"""
Tests checksum method
"""
self.ufc = UserFileCache()
checksum1 = self.ufc.checksum(
fileName=path.join(getTestBase(), "../data/ewv_crab_EwvAnalysis_31_111229_140959_publish.tgz")
)
checksum2 = self.ufc.checksum(
fileName=path.join(getTestBase(), "../data/ewv_crab_EwvAnalysis_31_resubmit_111229_144319_publish.tgz")
)
self.assertTrue(checksum1)
self.assertTrue(checksum2)
self.assertFalse(checksum1 == checksum2)
self.assertRaises(IOError, self.ufc.checksum, **{"fileName": "does_not_exist"})
return
def testUploadDownload(self):
if "UFCURL" in os.environ:
currdir = getTestBase()
upfile = path.join(currdir, "WMCore_t/Services_t/UserFileCache_t/test_file.tgz") # file to upload
ufc = UserFileCache({"endpoint": os.environ["UFCURL"]})
# named upload/download
res = ufc.upload(upfile, "name_publish.tgz")
ufc.download(name=res["name"], output="name_publish.tgz")
# hashkey upload/download
res = ufc.upload(upfile)
ufc.download(res["hashkey"], output="pippo_publish_down.tgz")
def __call__(self):
self.logger.info('Getting the tarball hash key')
tarballdir=glob.glob(self.requestarea+'/inputs/*.tgz')
if len(tarballdir) != 1 :
self.logger.info('%sError%s: Could not find tarball or there is more than one tarball'% (colors.RED, colors.NORMAL))
raise ConfigurationException
tarballdir=tarballdir[0]
#checking task status
self.logger.info('Checking task status')
server = HTTPRequests(self.serverurl, self.proxyfilename, self.proxyfilename, version=__version__)
dictresult, status, _ = server.get(self.uri, data = { 'workflow' : self.cachedinfo['RequestName'], 'verbose': 0 })
dictresult = dictresult['result'][0] #take just the significant part
if status != 200:
msg = "Problem retrieving status:\ninput:%s\noutput:%s\nreason:%s" % (str(self.cachedinfo['RequestName']), str(dictresult), str(reason))
raise RESTCommunicationException(msg)
self.logger.info('Task status: %s' % dictresult['status'])
accepstate = ['KILLED','FINISHED','FAILED','KILLFAILED', 'COMPLETED']
if dictresult['status'] not in accepstate:
msg = ('%sError%s: Only task with this status can be purge: {0}'.format(accepstate) % (colors.RED, colors.NORMAL))
raise ConfigurationException(msg)
#getting the cache url
if not self.options.scheddonly:
baseurl=self.getUrl(self.instance, resource='info')
cacheurl=server_info('backendurls', self.serverurl, self.proxyfilename, baseurl)
cacheurl=cacheurl['cacheSSL']
cacheurldict={'endpoint' : cacheurl, 'pycurl': True}
ufc = UserFileCache(cacheurldict)
hashkey = ufc.checksum(tarballdir)
self.logger.info('Tarball hashkey :%s' %hashkey)
self.logger.info('Attempting to clean user file cache')
ufcresult = ufc.removeFile(hashkey)
if ufcresult == '' :
self.logger.info('%sSuccess%s: Successfully remove file from cache' % (colors.GREEN, colors.NORMAL))
else:
self.logger.info('%sError%s: Failed to remove the file from cache' % (colors.RED, colors.NORMAL))
if not self.options.cacheonly:
self.logger.info('Getting the schedd address')
baseurl=self.getUrl(self.instance, resource='info')
try:
sceddaddress = server_info('scheddaddress', self.serverurl, self.proxyfilename, baseurl, workflow = self.cachedinfo['RequestName'] )
except HTTPException, he:
self.logger.info('%sError%s: Failed to get the schedd address' % (colors.RED, colors.NORMAL))
raise HTTPException,he
self.logger.debug('%sSuccess%s: Successfully getting schedd address' % (colors.GREEN, colors.NORMAL))
self.logger.debug('Schedd address: %s' % sceddaddress)
self.logger.info('Attempting to clean user file schedd')
gssishrm = 'gsissh -o ConnectTimeout=60 -o PasswordAuthentication=no ' + sceddaddress + ' rm -rf ' + self.cachedinfo['RequestName']
self.logger.debug('gsissh command: %s' % gssishrm)
delprocess=subprocess.Popen(gssishrm, stdout= subprocess.PIPE, stderr= subprocess.PIPE, shell=True)
stdout, stderr = delprocess.communicate()
exitcode = delprocess.returncode
if exitcode == 0 :
self.logger.info('%sSuccess%s: Successfully remove task from scehdd' % (colors.GREEN, colors.NORMAL))
else :
self.logger.info('%sError%s: Failed to remove task from schedd' % (colors.RED, colors.NORMAL))
self.logger.debug('gsissh stdout: %s\ngsissh stderr: %s\ngsissh exitcode: %s' % (stdout,stderr,exitcode))
def __call__(self):
self.logger.info('Getting the tarball hash key')
tarballdir = glob.glob(self.requestarea + '/inputs/*.tgz')
if len(tarballdir) != 1:
self.logger.info(
'%sError%s: Could not find tarball or there is more than one tarball'
% (colors.RED, colors.NORMAL))
raise ConfigurationException
tarballdir = tarballdir[0]
#checking task status
self.logger.info('Checking task status')
serverFactory = CRABClient.Emulator.getEmulator('rest')
server = serverFactory(self.serverurl,
self.proxyfilename,
self.proxyfilename,
version=__version__)
dictresult, status, _ = server.get(self.uri,
data={
'workflow':
self.cachedinfo['RequestName'],
'verbose':
0
})
dictresult = dictresult['result'][0] #take just the significant part
if status != 200:
msg = "Problem retrieving task status:\ninput: %s\noutput: %s\nreason: %s" % (
str(self.cachedinfo['RequestName']), str(dictresult),
str(reason))
raise RESTCommunicationException(msg)
self.logger.info('Task status: %s' % dictresult['status'])
accepstate = [
'KILLED', 'FINISHED', 'FAILED', 'KILLFAILED', 'COMPLETED'
]
if dictresult['status'] not in accepstate:
msg = ('%sError%s: Only tasks with these status can be purged: {0}'
.format(accepstate) % (colors.RED, colors.NORMAL))
raise ConfigurationException(msg)
#getting the cache url
cacheresult = {}
scheddresult = {}
gsisshdict = {}
if not self.options.scheddonly:
baseurl = getUrl(self.instance, resource='info')
cacheurl = server_info('backendurls', self.serverurl,
self.proxyfilename, baseurl)
cacheurl = cacheurl['cacheSSL']
cacheurldict = {'endpoint': cacheurl, 'pycurl': True}
ufc = UserFileCache(cacheurldict)
hashkey = ufc.checksum(tarballdir)
self.logger.info('Tarball hashkey: %s' % hashkey)
self.logger.info(
'Attempting to remove task file from crab server cache')
try:
ufcresult = ufc.removeFile(hashkey)
except HTTPException, re:
if re.headers.has_key(
'X-Error-Info'
) and 'Not such file' in re.headers['X-Error-Info']:
self.logger.info(
'%sError%s: Failed to find task file in crab server cache; the file might have been already purged'
% (colors.RED, colors.NORMAL))
raise HTTPException, re
if ufcresult == '':
self.logger.info(
'%sSuccess%s: Successfully removed task files from crab server cache'
% (colors.GREEN, colors.NORMAL))
cacheresult = 'SUCCESS'
else:
self.logger.info(
'%sError%s: Failed to remove task files from crab server cache'
% (colors.RED, colors.NORMAL))
cacheresult = 'FAILED'
