def __uploadFile(self, se, pfn): """proxied upload file""" res = self.__prepareSecurityDetails() if not res["OK"]: return res # Put file to the SE try: storageElement = StorageElement(se) except AttributeError as x: errStr = "__uploadFile: Exception while instantiating the Storage Element." gLogger.exception(errStr, se, str(x)) return S_ERROR(errStr) putFileDir = "%s/putFile" % BASE_PATH localFileName = "%s/%s" % (putFileDir, os.path.basename(pfn)) res = returnSingleResult(storageElement.putFile({pfn: localFileName})) if not res["OK"]: gLogger.error("prepareFile: Failed to put local file to storage.", res["Message"]) # Clear the local cache try: gLogger.debug("Removing temporary file", localFileName) os.remove(localFileName) except Exception as x: gLogger.exception("Failed to remove local file", localFileName, x) return res
def pushNewResults(): #cd to temp folder to temporary save the zip files os.chdir(temp_save_path) logger.warning('Checking results directory for new added zip files...') from DIRAC.Core.Base.Script import initialize #from DIRAC import gLogger #gLogger.setLevel("DEBUG") initialize(ignoreErrors = True, enableCommandLine = False) from DIRAC.Resources.Storage.StorageElement import StorageElement statSE = StorageElement(diracStorageElementName) #print diracStorageElementFolder print "Before listDirectory" dirDict = statSE.listDirectory(diracStorageElementFolder) print "After listDirectory" print dirDict for zipResult in dirDict['Value']['Successful'][diracStorageElementFolder]['Files']: fileName, fileExtension = os.path.splitext(zipResult) #get the File, copy the file to the current local directory res = statSE.getFile(os.path.join(diracStorageElementFolder, zipResult)) if not res['OK'] or ( res['OK'] and len(res['Value']['Failed']) > 0): logger.errot("Failed download of " + zipResult) continue results_list = AddedResults.objects.filter(identifier__exact=fileName) res = True if not results_list: logger.info('New zip: {0}, found in results directory, calling pushZip command...'.format(zipResult)) res = pushZip.pushThis(os.path.join(temp_save_path, zipResult )) if not res: logger.error("Error pushing results, not removing") continue #remove it from the upload_test folder statSE.removeFile(os.path.join(diracStorageElementFolder, zipResult)) #put the file into the added folder statSE.putFile({ os.path.join(addedDiracStorageFolder, zipResult) : zipResult}) #also remove the file from the current directory os.remove(os.path.join(temp_save_path, zipResult))
def sendViaDiracStorageElement(zipFile): head, tailzipFile = os.path.split(zipFile) from DIRAC.Core.Base.Script import parseCommandLine, initialize initialize(ignoreErrors = True, enableCommandLine = False) from DIRAC.Resources.Storage.StorageElement import StorageElement statSE = StorageElement(diracStorageElementName) log = statSE.putFile({ os.path.join(diracStorageElementFolder, tailzipFile) : zipFile}) logger.info('{0}'.format(log))
def test_putFile(setuptest): """ Copy a file """ # XXX: this is not good ! # The mock I use for S3 seem to have a bug uploading files # with presigned URL. So for the time being, I upload directly, # but this should be checked # https://github.com/adobe/S3Mock/issues/219 se = StorageElement('S3-DIRECT') res = se.putFile(putFile) assert res['OK'], res for lfn in putFile: assert lfn in res['Value']['Successful']