def testUploadDownload(self):
if 'UFCURL' in os.environ:
currdir = getTestBase()
upfile = path.join(currdir, 'WMCore_t/Services_t/UserFileCache_t/test_file.tgz') #file to upload
ufc = UserFileCache({'endpoint':os.environ['UFCURL']})
#named upload/download
res = ufc.upload(upfile, 'name_publish.tgz')
ufc.download(name=res['name'], output='name_publish.tgz')
#hashkey upload/download
res = ufc.upload(upfile)
ufc.download(res['hashkey'], output='pippo_publish_down.tgz')
def testUploadDownload(self):
if "UFCURL" in os.environ:
currdir = getTestBase()
upfile = path.join(currdir, "WMCore_t/Services_t/UserFileCache_t/test_file.tgz") # file to upload
ufc = UserFileCache({"endpoint": os.environ["UFCURL"]})
# named upload/download
res = ufc.upload(upfile, "name_publish.tgz")
ufc.download(name=res["name"], output="name_publish.tgz")
# hashkey upload/download
res = ufc.upload(upfile)
ufc.download(res["hashkey"], output="pippo_publish_down.tgz")
def testUploadDownload(self):
if 'UFCURL' in os.environ:
currdir = getTestBase()
upfile = path.join(
currdir, 'WMCore_t/Services_t/UserFileCache_t/test_file.tgz'
) #file to upload
upfileLog = path.join(
currdir, 'WMCore_t/Services_t/UserFileCache_t/uplog.txt'
) #file to upload
ufc = UserFileCache({
'endpoint': os.environ['UFCURL'],
'pycurl': True
})
#hashkey upload/download
res = ufc.upload(upfile)
ufc.download(res['hashkey'], output='pippo_publish_down.tgz')
#hashkey deletion
ufc.removeFile(res['hashkey'])
#log upload/download
res = ufc.uploadLog(upfileLog)
ufc.downloadLog(upfileLog, upfileLog + '.downloaded')
self.assertTrue(filecmp.cmp(upfileLog, upfileLog + '.downloaded'))
def uploadPublishWorkflow(config, workflow, ufcEndpoint, workDir):
"""
Write out and upload to the UFC a JSON file
with all the info needed to publish this dataset later
"""
retok, proxyfile = getProxy(config, workflow.dn, workflow.vogroup, workflow.vorole)
if not retok:
logging.info("Cannot get the user's proxy")
return False
ufc = UserFileCache({'endpoint': ufcEndpoint, 'cert': proxyfile, 'key': proxyfile})
# Skip tasks ending in LogCollect, they have nothing interesting.
taskNameParts = workflow.task.split('/')
if taskNameParts.pop() in ['LogCollect']:
logging.info('Skipping LogCollect task')
return False
logging.info('Generating JSON for publication of %s of type %s' % (workflow.name, workflow.wfType))
myThread = threading.currentThread()
dbsDaoFactory = DAOFactory(package = "WMComponent.DBS3Buffer",
logger = myThread.logger, dbinterface = myThread.dbi)
findFiles = dbsDaoFactory(classname = "LoadFilesByWorkflow")
# Fetch and filter the files to the ones we actually need
uploadDatasets = {}
uploadFiles = findFiles.execute(workflowName = workflow.name)
for file in uploadFiles:
datasetName = file['datasetPath']
if not uploadDatasets.has_key(datasetName):
uploadDatasets[datasetName] = []
uploadDatasets[datasetName].append(file)
if not uploadDatasets:
logging.info('No datasets found to upload.')
return False
# Write JSON file and then create tarball with it
baseName = '%s_publish.tgz' % workflow.name
jsonName = os.path.join(workDir, '%s_publish.json' % workflow.name)
tgzName = os.path.join(workDir, baseName)
with open(jsonName, 'w') as jsonFile:
json.dump(uploadDatasets, fp=jsonFile, cls=FileEncoder, indent=2)
# Only in 2.7 does tarfile become usable as context manager
tgzFile = tarfile.open(name=tgzName, mode='w:gz')
tgzFile.add(jsonName)
tgzFile.close()
result = ufc.upload(fileName=tgzName, name=baseName)
logging.debug('Upload result %s' % result)
# If this doesn't work, exception will propogate up and block archiving the task
logging.info('Uploaded with name %s and hashkey %s' % (result['name'], result['hashkey']))
return
def upload(self):
"""
Upload the tarball to the File Cache
"""
self.close()
archiveName = self.tarfile.name
serverUrl = ""
self.logger.debug(" uploading archive to cache %s " % archiveName)
ufc = UserFileCache({'endpoint' : self.config.JobType.filecacheurl})
result = ufc.upload(archiveName)
if 'hashkey' not in result:
self.logger.error("Failed to upload source files: %s" % str(result))
raise CachefileNotFoundException
return self.config.JobType.filecacheurl, str(result['hashkey']) + '.tar.gz', self.checksum
def testUploadDownload(self):
if 'UFCURL' in os.environ:
currdir = getTestBase()
upfile = path.join(currdir, 'WMCore_t/Services_t/UserFileCache_t/test_file.tgz') #file to upload
upfileLog = path.join(currdir, 'WMCore_t/Services_t/UserFileCache_t/uplog.txt') #file to upload
ufc = UserFileCache({'endpoint':os.environ['UFCURL']})
#hashkey upload/download
res = ufc.upload(upfile)
ufc.download(res['hashkey'], output='pippo_publish_down.tgz')
#log upload/download
res = ufc.uploadLog(upfileLog)
ufc.downloadLog(upfileLog, upfileLog+'.downloaded')
self.assertTrue(filecmp.cmp(upfileLog, upfileLog+'.downloaded'))
def testUploadDownload(self):
if "UFCURL" in os.environ:
currdir = getTestBase()
upfile = path.join(currdir, "WMCore_t/Services_t/UserFileCache_t/test_file.tgz") # file to upload
upfileLog = path.join(currdir, "WMCore_t/Services_t/UserFileCache_t/uplog.txt") # file to upload
ufc = UserFileCache({"endpoint": os.environ["UFCURL"], "pycurl": True})
# hashkey upload/download
res = ufc.upload(upfile)
ufc.download(res["hashkey"], output="pippo_publish_down.tgz")
# hashkey deletion
ufc.removeFile(res["hashkey"])
# log upload/download
res = ufc.uploadLog(upfileLog)
ufc.downloadLog(upfileLog, upfileLog + ".downloaded")
self.assertTrue(filecmp.cmp(upfileLog, upfileLog + ".downloaded"))
def uploadSandboxToUFC( self, url, sandbox = None, name = "YAATFile" ):
if not sandbox:
sandbox = self.sandbox
req = UserFileCache( { 'endpoint' : url} )
return req.upload( sandbox, name )
def uploadPublishWorkflow(config, workflow, ufcEndpoint, workDir):
"""
Write out and upload to the UFC a JSON file
with all the info needed to publish this dataset later
"""
retok, proxyfile = getProxy(config, workflow.dn, workflow.vogroup,
workflow.vorole)
if not retok:
logging.info("Cannot get the user's proxy")
return False
ufc = UserFileCache({
'endpoint': ufcEndpoint,
'cert': proxyfile,
'key': proxyfile
})
# Skip tasks ending in LogCollect, they have nothing interesting.
taskNameParts = workflow.task.split('/')
if taskNameParts.pop() in ['LogCollect']:
logging.info('Skipping LogCollect task')
return False
logging.info('Generating JSON for publication of %s of type %s' %
(workflow.name, workflow.wfType))
myThread = threading.currentThread()
dbsDaoFactory = DAOFactory(package="WMComponent.DBS3Buffer",
logger=myThread.logger,
dbinterface=myThread.dbi)
findFiles = dbsDaoFactory(classname="LoadFilesByWorkflow")
# Fetch and filter the files to the ones we actually need
uploadDatasets = {}
uploadFiles = findFiles.execute(workflowName=workflow.name)
for file in uploadFiles:
datasetName = file['datasetPath']
if not uploadDatasets.has_key(datasetName):
uploadDatasets[datasetName] = []
uploadDatasets[datasetName].append(file)
if not uploadDatasets:
logging.info('No datasets found to upload.')
return False
# Write JSON file and then create tarball with it
baseName = '%s_publish.tgz' % workflow.name
jsonName = os.path.join(workDir, '%s_publish.json' % workflow.name)
tgzName = os.path.join(workDir, baseName)
with open(jsonName, 'w') as jsonFile:
json.dump(uploadDatasets, fp=jsonFile, cls=FileEncoder, indent=2)
# Only in 2.7 does tarfile become usable as context manager
tgzFile = tarfile.open(name=tgzName, mode='w:gz')
tgzFile.add(jsonName)
tgzFile.close()
result = ufc.upload(fileName=tgzName, name=baseName)
logging.debug('Upload result %s' % result)
# If this doesn't work, exception will propogate up and block archiving the task
logging.info('Uploaded with name %s and hashkey %s' %
(result['name'], result['hashkey']))
return
