def __uploadFile(self, se, pfn):
"""proxied upload file"""
res = self.__prepareSecurityDetails()
if not res["OK"]:
return res
# Put file to the SE
try:
storageElement = StorageElement(se)
except AttributeError as x:
errStr = "__uploadFile: Exception while instantiating the Storage Element."
gLogger.exception(errStr, se, str(x))
return S_ERROR(errStr)
putFileDir = "%s/putFile" % BASE_PATH
localFileName = "%s/%s" % (putFileDir, os.path.basename(pfn))
res = returnSingleResult(storageElement.putFile({pfn: localFileName}))
if not res["OK"]:
gLogger.error("prepareFile: Failed to put local file to storage.",
res["Message"])
# Clear the local cache
try:
gLogger.debug("Removing temporary file", localFileName)
os.remove(localFileName)
except Exception as x:
gLogger.exception("Failed to remove local file", localFileName, x)
return res
def pushNewResults():
#cd to temp folder to temporary save the zip files
os.chdir(temp_save_path)
logger.warning('Checking results directory for new added zip files...')
from DIRAC.Core.Base.Script import initialize
#from DIRAC import gLogger
#gLogger.setLevel("DEBUG")
initialize(ignoreErrors = True, enableCommandLine = False)
from DIRAC.Resources.Storage.StorageElement import StorageElement
statSE = StorageElement(diracStorageElementName)
#print diracStorageElementFolder
print "Before listDirectory"
dirDict = statSE.listDirectory(diracStorageElementFolder)
print "After listDirectory"
print dirDict
for zipResult in dirDict['Value']['Successful'][diracStorageElementFolder]['Files']:
fileName, fileExtension = os.path.splitext(zipResult)
#get the File, copy the file to the current local directory
res = statSE.getFile(os.path.join(diracStorageElementFolder, zipResult))
if not res['OK'] or ( res['OK'] and len(res['Value']['Failed']) > 0):
logger.errot("Failed download of " + zipResult)
continue
results_list = AddedResults.objects.filter(identifier__exact=fileName)
res = True
if not results_list:
logger.info('New zip: {0}, found in results directory, calling pushZip command...'.format(zipResult))
res = pushZip.pushThis(os.path.join(temp_save_path, zipResult ))
if not res:
logger.error("Error pushing results, not removing")
continue
#remove it from the upload_test folder
statSE.removeFile(os.path.join(diracStorageElementFolder, zipResult))
#put the file into the added folder
statSE.putFile({ os.path.join(addedDiracStorageFolder, zipResult) : zipResult})
#also remove the file from the current directory
os.remove(os.path.join(temp_save_path, zipResult))
def sendViaDiracStorageElement(zipFile):
head, tailzipFile = os.path.split(zipFile)
from DIRAC.Core.Base.Script import parseCommandLine, initialize
initialize(ignoreErrors = True, enableCommandLine = False)
from DIRAC.Resources.Storage.StorageElement import StorageElement
statSE = StorageElement(diracStorageElementName)
log = statSE.putFile({ os.path.join(diracStorageElementFolder, tailzipFile) : zipFile})
logger.info('{0}'.format(log))
def test_putFile(setuptest):
""" Copy a file """
# XXX: this is not good !
# The mock I use for S3 seem to have a bug uploading files
# with presigned URL. So for the time being, I upload directly,
# but this should be checked
# https://github.com/adobe/S3Mock/issues/219
se = StorageElement('S3-DIRECT')
res = se.putFile(putFile)
assert res['OK'], res
for lfn in putFile:
assert lfn in res['Value']['Successful']
