def testUploadDownload(self): if 'UFCURL' in os.environ: currdir = getTestBase() upfile = path.join( currdir, 'WMCore_t/Services_t/UserFileCache_t/test_file.tgz' ) #file to upload upfileLog = path.join( currdir, 'WMCore_t/Services_t/UserFileCache_t/uplog.txt' ) #file to upload ufc = UserFileCache({ 'endpoint': os.environ['UFCURL'], 'pycurl': True }) #hashkey upload/download res = ufc.upload(upfile) ufc.download(res['hashkey'], output='pippo_publish_down.tgz') #hashkey deletion ufc.removeFile(res['hashkey']) #log upload/download res = ufc.uploadLog(upfileLog) ufc.downloadLog(upfileLog, upfileLog + '.downloaded') self.assertTrue(filecmp.cmp(upfileLog, upfileLog + '.downloaded'))
def testUploadDownload(self): if 'UFCURL' in os.environ: currdir = getTestBase() upfile = path.join(currdir, 'WMCore_t/Services_t/UserFileCache_t/test_file.tgz') #file to upload upfileLog = path.join(currdir, 'WMCore_t/Services_t/UserFileCache_t/uplog.txt') #file to upload ufc = UserFileCache({'endpoint':os.environ['UFCURL'], 'pycurl': True}) #hashkey upload/download res = ufc.upload(upfile) ufc.download(res['hashkey'], output='pippo_publish_down.tgz') #hashkey deletion ufc.removeFile(res['hashkey']) #log upload/download res = ufc.uploadLog(upfileLog) ufc.downloadLog(upfileLog, upfileLog+'.downloaded') self.assertTrue(filecmp.cmp(upfileLog, upfileLog+'.downloaded'))
def __call__(self): self.logger.info('Getting the tarball hash key') tarballdir = glob.glob(self.requestarea+'/inputs/*.tgz') if len(tarballdir) != 1: self.logger.info('%sError%s: Could not find tarball or there is more than one tarball'% (colors.RED, colors.NORMAL)) raise ConfigurationException tarballdir = tarballdir[0] #checking task status self.logger.info('Checking task status') serverFactory = CRABClient.Emulator.getEmulator('rest') server = serverFactory(self.serverurl, self.proxyfilename, self.proxyfilename, version=__version__) dictresult, status, _ = server.get(self.uri, data = {'workflow': self.cachedinfo['RequestName'], 'verbose': 0}) dictresult = dictresult['result'][0] #take just the significant part if status != 200: msg = "Problem retrieving task status:\ninput: %s\noutput: %s\nreason: %s" % (str(self.cachedinfo['RequestName']), str(dictresult), str(reason)) raise RESTCommunicationException(msg) self.logger.info('Task status: %s' % dictresult['status']) accepstate = ['KILLED','FINISHED','FAILED','KILLFAILED', 'COMPLETED'] if dictresult['status'] not in accepstate: msg = ('%sError%s: Only tasks with these status can be purged: {0}'.format(accepstate) % (colors.RED, colors.NORMAL)) raise ConfigurationException(msg) #getting the cache url cacheresult = {} scheddresult = {} gsisshdict = {} if not self.options.scheddonly: baseurl = getUrl(self.instance, resource='info') cacheurl = server_info('backendurls', self.serverurl, self.proxyfilename, baseurl) cacheurl = cacheurl['cacheSSL'] cacheurldict = {'endpoint': cacheurl, 'pycurl': True} ufc = UserFileCache(cacheurldict) hashkey = ufc.checksum(tarballdir) self.logger.info('Tarball hashkey: %s' %hashkey) self.logger.info('Attempting to remove task file from crab server cache') try: ufcresult = ufc.removeFile(hashkey) except HTTPException, re: if re.headers.has_key('X-Error-Info') and 'Not such file' in re.headers['X-Error-Info']: self.logger.info('%sError%s: Failed to find task file in crab server cache; the file might have been already purged' % (colors.RED,colors.NORMAL)) raise HTTPException , re if ufcresult == '': self.logger.info('%sSuccess%s: Successfully removed task files from crab server cache' % (colors.GREEN, colors.NORMAL)) cacheresult = 'SUCCESS' else: self.logger.info('%sError%s: Failed to remove task files from crab server cache' % (colors.RED, colors.NORMAL)) cacheresult = 'FAILED'
def testUploadDownload(self): if "UFCURL" in os.environ: currdir = getTestBase() upfile = path.join(currdir, "WMCore_t/Services_t/UserFileCache_t/test_file.tgz") # file to upload upfileLog = path.join(currdir, "WMCore_t/Services_t/UserFileCache_t/uplog.txt") # file to upload ufc = UserFileCache({"endpoint": os.environ["UFCURL"], "pycurl": True}) # hashkey upload/download res = ufc.upload(upfile) ufc.download(res["hashkey"], output="pippo_publish_down.tgz") # hashkey deletion ufc.removeFile(res["hashkey"]) # log upload/download res = ufc.uploadLog(upfileLog) ufc.downloadLog(upfileLog, upfileLog + ".downloaded") self.assertTrue(filecmp.cmp(upfileLog, upfileLog + ".downloaded"))
def __call__(self): self.logger.info('Getting the tarball hash key') tarballdir=glob.glob(self.requestarea+'/inputs/*.tgz') if len(tarballdir) != 1 : self.logger.info('%sError%s: Could not find tarball or there is more than one tarball'% (colors.RED, colors.NORMAL)) raise ConfigurationException tarballdir=tarballdir[0] #checking task status self.logger.info('Checking task status') server = HTTPRequests(self.serverurl, self.proxyfilename, self.proxyfilename, version=__version__) dictresult, status, _ = server.get(self.uri, data = { 'workflow' : self.cachedinfo['RequestName'], 'verbose': 0 }) dictresult = dictresult['result'][0] #take just the significant part if status != 200: msg = "Problem retrieving status:\ninput:%s\noutput:%s\nreason:%s" % (str(self.cachedinfo['RequestName']), str(dictresult), str(reason)) raise RESTCommunicationException(msg) self.logger.info('Task status: %s' % dictresult['status']) accepstate = ['KILLED','FINISHED','FAILED','KILLFAILED', 'COMPLETED'] if dictresult['status'] not in accepstate: msg = ('%sError%s: Only task with this status can be purge: {0}'.format(accepstate) % (colors.RED, colors.NORMAL)) raise ConfigurationException(msg) #getting the cache url if not self.options.scheddonly: baseurl=self.getUrl(self.instance, resource='info') cacheurl=server_info('backendurls', self.serverurl, self.proxyfilename, baseurl) cacheurl=cacheurl['cacheSSL'] cacheurldict={'endpoint' : cacheurl, 'pycurl': True} ufc = UserFileCache(cacheurldict) hashkey = ufc.checksum(tarballdir) self.logger.info('Tarball hashkey :%s' %hashkey) self.logger.info('Attempting to clean user file cache') ufcresult = ufc.removeFile(hashkey) if ufcresult == '' : self.logger.info('%sSuccess%s: Successfully remove file from cache' % (colors.GREEN, colors.NORMAL)) else: self.logger.info('%sError%s: Failed to remove the file from cache' % (colors.RED, colors.NORMAL)) if not self.options.cacheonly: self.logger.info('Getting the schedd address') baseurl=self.getUrl(self.instance, resource='info') try: sceddaddress = server_info('scheddaddress', self.serverurl, self.proxyfilename, baseurl, workflow = self.cachedinfo['RequestName'] ) except HTTPException, he: self.logger.info('%sError%s: Failed to get the schedd address' % (colors.RED, colors.NORMAL)) raise HTTPException,he self.logger.debug('%sSuccess%s: Successfully getting schedd address' % (colors.GREEN, colors.NORMAL)) self.logger.debug('Schedd address: %s' % sceddaddress) self.logger.info('Attempting to clean user file schedd') gssishrm = 'gsissh -o ConnectTimeout=60 -o PasswordAuthentication=no ' + sceddaddress + ' rm -rf ' + self.cachedinfo['RequestName'] self.logger.debug('gsissh command: %s' % gssishrm) delprocess=subprocess.Popen(gssishrm, stdout= subprocess.PIPE, stderr= subprocess.PIPE, shell=True) stdout, stderr = delprocess.communicate() exitcode = delprocess.returncode if exitcode == 0 : self.logger.info('%sSuccess%s: Successfully remove task from scehdd' % (colors.GREEN, colors.NORMAL)) else : self.logger.info('%sError%s: Failed to remove task from schedd' % (colors.RED, colors.NORMAL)) self.logger.debug('gsissh stdout: %s\ngsissh stderr: %s\ngsissh exitcode: %s' % (stdout,stderr,exitcode))
def __call__(self): self.logger.info('Getting the tarball hash key') tarballdir = glob.glob(self.requestarea + '/inputs/*.tgz') if len(tarballdir) != 1: self.logger.info( '%sError%s: Could not find tarball or there is more than one tarball' % (colors.RED, colors.NORMAL)) raise ConfigurationException tarballdir = tarballdir[0] #checking task status self.logger.info('Checking task status') serverFactory = CRABClient.Emulator.getEmulator('rest') server = serverFactory(self.serverurl, self.proxyfilename, self.proxyfilename, version=__version__) dictresult, status, _ = server.get(self.uri, data={ 'workflow': self.cachedinfo['RequestName'], 'verbose': 0 }) dictresult = dictresult['result'][0] #take just the significant part if status != 200: msg = "Problem retrieving task status:\ninput: %s\noutput: %s\nreason: %s" % ( str(self.cachedinfo['RequestName']), str(dictresult), str(reason)) raise RESTCommunicationException(msg) self.logger.info('Task status: %s' % dictresult['status']) accepstate = [ 'KILLED', 'FINISHED', 'FAILED', 'KILLFAILED', 'COMPLETED' ] if dictresult['status'] not in accepstate: msg = ('%sError%s: Only tasks with these status can be purged: {0}' .format(accepstate) % (colors.RED, colors.NORMAL)) raise ConfigurationException(msg) #getting the cache url cacheresult = {} scheddresult = {} gsisshdict = {} if not self.options.scheddonly: baseurl = getUrl(self.instance, resource='info') cacheurl = server_info('backendurls', self.serverurl, self.proxyfilename, baseurl) cacheurl = cacheurl['cacheSSL'] cacheurldict = {'endpoint': cacheurl, 'pycurl': True} ufc = UserFileCache(cacheurldict) hashkey = ufc.checksum(tarballdir) self.logger.info('Tarball hashkey: %s' % hashkey) self.logger.info( 'Attempting to remove task file from crab server cache') try: ufcresult = ufc.removeFile(hashkey) except HTTPException, re: if re.headers.has_key( 'X-Error-Info' ) and 'Not such file' in re.headers['X-Error-Info']: self.logger.info( '%sError%s: Failed to find task file in crab server cache; the file might have been already purged' % (colors.RED, colors.NORMAL)) raise HTTPException, re if ufcresult == '': self.logger.info( '%sSuccess%s: Successfully removed task files from crab server cache' % (colors.GREEN, colors.NORMAL)) cacheresult = 'SUCCESS' else: self.logger.info( '%sError%s: Failed to remove task files from crab server cache' % (colors.RED, colors.NORMAL)) cacheresult = 'FAILED'
def __call__(self): self.logger.info('Getting the tarball hash key') inputlist = {'subresource': 'search', 'workflow': self.cachedinfo['RequestName']} serverFactory = CRABClient.Emulator.getEmulator('rest') server = serverFactory(self.serverurl, self.proxyfilename, self.proxyfilename, version=__version__) uri = getUrl(self.instance, resource = 'task') dictresult, _, _ = server.get(uri, data = inputlist) tm_user_sandbox = getColumn(dictresult, 'tm_user_sandbox') hashkey = tm_user_sandbox.replace(".tar.gz","") # Get the schedd address from the DB info and strip off the 'crab3@' prefix if it exists scheddaddress = getColumn(dictresult, 'tm_schedd') scheddaddress = scheddaddress.split('@')[1] if '@' in scheddaddress else scheddaddress self.logger.info('Checking task status') serverFactory = CRABClient.Emulator.getEmulator('rest') server = serverFactory(self.serverurl, self.proxyfilename, self.proxyfilename, version=__version__) dictresult, _, _ = server.get(self.uri, data = {'workflow': self.cachedinfo['RequestName'], 'verbose': 0}) dictresult = dictresult['result'][0] #take just the significant part self.logger.info('Task status: %s' % dictresult['status']) accepstate = ['SUBMITFAILED','KILLED','FINISHED','FAILED','KILLFAILED', 'COMPLETED'] if dictresult['status'] not in accepstate: msg = ('%sError%s: Only tasks with these status can be purged: {0}'.format(accepstate) % (colors.RED, colors.NORMAL)) raise ConfigurationException(msg) #getting the cache url cacheresult = {} scheddresult = {} gsisshdict = {} if not self.options.scheddonly: baseurl = getUrl(self.instance, resource='info') cacheurl = server_info('backendurls', self.serverurl, self.proxyfilename, baseurl) cacheurl = cacheurl['cacheSSL'] cacheurldict = {'endpoint': cacheurl, 'pycurl': True} ufc = UserFileCache(cacheurldict) self.logger.info('Tarball hashkey: %s' %hashkey) self.logger.info('Attempting to remove task file from crab server cache') try: ufcresult = ufc.removeFile(hashkey) except HTTPException as re: if 'X-Error-Info' in re.headers and 'Not such file' in re.headers['X-Error-Info']: self.logger.info('%sError%s: Failed to find task file in crab server cache; the file might have been already purged' % (colors.RED,colors.NORMAL)) raise if ufcresult == '': self.logger.info('%sSuccess%s: Successfully removed task files from crab server cache' % (colors.GREEN, colors.NORMAL)) cacheresult = 'SUCCESS' else: self.logger.info('%sError%s: Failed to remove task files from crab server cache' % (colors.RED, colors.NORMAL)) cacheresult = 'FAILED' if not self.options.cacheonly: self.logger.debug('%sSuccess%s: Successfully got schedd address' % (colors.GREEN, colors.NORMAL)) self.logger.debug('Schedd address: %s' % scheddaddress) self.logger.info('Attempting to remove task from schedd') gssishrm = 'gsissh -o ConnectTimeout=60 -o PasswordAuthentication=no ' + scheddaddress + ' rm -rf ' + self.cachedinfo['RequestName'] self.logger.debug('gsissh command: %s' % gssishrm) delprocess=subprocess.Popen(gssishrm, stdout= subprocess.PIPE, stderr= subprocess.PIPE, shell=True) stdout, stderr = delprocess.communicate() exitcode = delprocess.returncode if exitcode == 0 : self.logger.info('%sSuccess%s: Successfully removed task from schedd' % (colors.GREEN, colors.NORMAL)) scheddresult = 'SUCCESS' gsisshdict = {} else : self.logger.info('%sError%s: Failed to remove task from schedd' % (colors.RED, colors.NORMAL)) scheddaddress = 'FAILED' self.logger.debug('gsissh stdout: %s\ngsissh stderr: %s\ngsissh exitcode: %s' % (stdout,stderr,exitcode)) gsisshdict = {'stdout' : stdout, 'stderr' : stderr , 'exitcode' : exitcode} return {'cacheresult' : cacheresult , 'scheddresult' : scheddresult , 'gsiresult' : gsisshdict}
def __call__(self): self.logger.info('Getting the tarball hash key') inputlist = {'subresource': 'search', 'workflow': self.cachedinfo['RequestName']} serverFactory = CRABClient.Emulator.getEmulator('rest') server = serverFactory(self.serverurl, self.proxyfilename, self.proxyfilename, version=__version__) uri = self.getUrl(self.instance, resource = 'task') dictresult, _, _ = server.get(uri, data = inputlist) tm_user_sandbox = getColumn(dictresult, 'tm_user_sandbox') hashkey = tm_user_sandbox.replace(".tar.gz","") # Get the schedd address from the DB info and strip off the 'crab3@' prefix if it exists scheddaddress = getColumn(dictresult, 'tm_schedd') scheddaddress = scheddaddress.split('@')[1] if '@' in scheddaddress else scheddaddress self.logger.info('Checking task status') serverFactory = CRABClient.Emulator.getEmulator('rest') server = serverFactory(self.serverurl, self.proxyfilename, self.proxyfilename, version=__version__) dictresult, _, _ = server.get(self.uri, data = {'workflow': self.cachedinfo['RequestName'], 'verbose': 0}) dictresult = dictresult['result'][0] #take just the significant part self.logger.info('Task status: %s' % dictresult['status']) accepstate = ['SUBMITFAILED','KILLED','FINISHED','FAILED','KILLFAILED', 'COMPLETED'] if dictresult['status'] not in accepstate: msg = ('%sError%s: Only tasks with these status can be purged: {0}'.format(accepstate) % (colors.RED, colors.NORMAL)) raise ConfigurationException(msg) #getting the cache url cacheresult = {} scheddresult = {} gsisshdict = {} if not self.options.scheddonly: baseurl = getUrl(self.instance, resource='info') cacheurl = server_info('backendurls', self.serverurl, self.proxyfilename, baseurl) cacheurl = cacheurl['cacheSSL'] cacheurldict = {'endpoint': cacheurl, 'pycurl': True} ufc = UserFileCache(cacheurldict) self.logger.info('Tarball hashkey: %s' %hashkey) self.logger.info('Attempting to remove task file from crab server cache') try: ufcresult = ufc.removeFile(hashkey) except HTTPException as re: if 'X-Error-Info' in re.headers and 'Not such file' in re.headers['X-Error-Info']: self.logger.info('%sError%s: Failed to find task file in crab server cache; the file might have been already purged' % (colors.RED,colors.NORMAL)) raise if ufcresult == '': self.logger.info('%sSuccess%s: Successfully removed task files from crab server cache' % (colors.GREEN, colors.NORMAL)) cacheresult = 'SUCCESS' else: self.logger.info('%sError%s: Failed to remove task files from crab server cache' % (colors.RED, colors.NORMAL)) cacheresult = 'FAILED' if not self.options.cacheonly: self.logger.debug('%sSuccess%s: Successfully got schedd address' % (colors.GREEN, colors.NORMAL)) self.logger.debug('Schedd address: %s' % scheddaddress) self.logger.info('Attempting to remove task from schedd') gssishrm = 'gsissh -o ConnectTimeout=60 -o PasswordAuthentication=no ' + scheddaddress + ' rm -rf ' + self.cachedinfo['RequestName'] self.logger.debug('gsissh command: %s' % gssishrm) delprocess=subprocess.Popen(gssishrm, stdout= subprocess.PIPE, stderr= subprocess.PIPE, shell=True) stdout, stderr = delprocess.communicate() exitcode = delprocess.returncode if exitcode == 0 : self.logger.info('%sSuccess%s: Successfully removed task from schedd' % (colors.GREEN, colors.NORMAL)) scheddresult = 'SUCCESS' gsisshdict = {} else : self.logger.info('%sError%s: Failed to remove task from schedd' % (colors.RED, colors.NORMAL)) scheddaddress = 'FAILED' self.logger.debug('gsissh stdout: %s\ngsissh stderr: %s\ngsissh exitcode: %s' % (stdout,stderr,exitcode)) gsisshdict = {'stdout' : stdout, 'stderr' : stderr , 'exitcode' : exitcode} return {'cacheresult' : cacheresult , 'scheddresult' : scheddresult , 'gsiresult' : gsisshdict}