def _presigned_putFile(self, urls, sourceSize=0):
"""Upload a local file.
..warning:: no 3rd party copy possible
:param urls: dictionary { urls : localFile }
:param sourceSize: size of the file in byte. Mandatory for third party copy (WHY ???)
Also, this parameter makes it essentially a non bulk operation for
third party copy, unless all files have the same size...
:returns: * Successful dict: { path : size }
* Failed dict: { path : error message }
* S_ERROR in case of argument problems
"""
log = LOG.getSubLogger("putFile")
failed = {}
successful = {}
# Construct a dict <url:{x-amz-meta-checksum: adler32}>
# it needs to be passed to createPresignedUrl
urlAdlers = {url: {"x-amz-meta-checksum": fileAdler(src_file)} for url, src_file in urls.items()}
res = self.S3GatewayClient.createPresignedUrl(self.name, "put_object", urlAdlers)
if not res["OK"]:
return res
failed.update(res["Value"]["Failed"])
# Contains <url: presignedResponse>
presignedResponses = res["Value"]["Successful"]
for dest_url, presignedResponse in presignedResponses.items():
src_file = urls[dest_url]
try:
cks = fileAdler(src_file)
if not cks:
log.warn("Cannot get ADLER32 checksum for %s" % src_file)
presignedURL = presignedResponse["url"]
presignedFields = presignedResponse["fields"]
with open(src_file, "rb") as src_fd:
# files = {'file': (dest_key, src_fd)}
files = {"file": src_fd}
response = requests.post(presignedURL, data=presignedFields, files=files)
if not response.ok:
raise Exception(response.reason)
successful[dest_url] = os.path.getsize(src_file)
except Exception as e:
failed[dest_url] = repr(e)
return S_OK({"Failed": failed, "Successful": successful})
def getFileMetadata(self, candidateFiles):
"""Returns the candidate file dictionary with associated metadata.
:param dict candidateFiles: The input candidate files dictionary has the structure: {'lfn':'','path':'','workflowSE':''}
This also assumes the files are in the current working directory.
:return: S_OK with File Metadata, S_ERROR
"""
#Retrieve the POOL File GUID(s) for any final output files
self.log.info('Will search for POOL GUIDs for: %s' %(', '.join(candidateFiles.keys())))
pfnGUIDs = {}
generated = []
for fname in candidateFiles.keys():
guid = makeGuid(fname)
pfnGUIDs[fname] = guid
generated.append(fname)
pfnGUID = S_OK(pfnGUIDs)
pfnGUID['generated'] = generated
#pfnGUID = getGUID(candidateFiles.keys())
#if not pfnGUID['OK']:
# self.log.error('PoolXMLFile failed to determine POOL GUID(s) for output file list, these will be generated by \
# the DataManager',pfnGUID['Message'])
# for fileName in candidateFiles.keys():
# candidateFiles[fileName]['guid']=''
#if pfnGUID['generated']:
self.log.debug('Generated GUID(s) for the following files ', ', '.join(pfnGUID['generated']))
#else:
# self.log.info('GUIDs found for all specified POOL files: %s' %(string.join(candidateFiles.keys(),', ')))
for pfn, guid in pfnGUID['Value'].items():
candidateFiles[pfn]['GUID'] = guid
#Get all additional metadata about the file necessary for requests
final = {}
for fileName, metadata in candidateFiles.items():
fileDict = {}
fileDict['LFN'] = metadata['lfn']
fileDict['Size'] = os.path.getsize(fileName)
fileDict['Addler'] = fileAdler(fileName)
fileDict['ADLER32'] = fileAdler(fileName)
fileDict['Checksum'] = fileAdler(fileName)
fileDict['ChecksumType'] = "ADLER32"
fileDict['GUID'] = metadata['GUID']
fileDict['Status'] = "Waiting"
final[fileName] = metadata
final[fileName]['filedict'] = fileDict
final[fileName]['localpath'] = '%s/%s' % (os.getcwd(), fileName)
#Sanity check all final candidate metadata keys are present (return S_ERROR if not)
mandatoryKeys = ['GUID', 'filedict'] #filedict is used for requests (this method adds guid and filedict)
for fileName, metadata in final.items():
for key in mandatoryKeys:
if key not in metadata:
return S_ERROR('File %s has missing %s' % (fileName, key))
return S_OK(final)
def getFileMetadata(self, candidateFiles):
"""Returns the candidate file dictionary with associated metadata.
:param dict candidateFiles: The input candidate files dictionary has the structure: {'lfn':'','path':'','workflowSE':''}
This also assumes the files are in the current working directory.
:return: S_OK with File Metadata, S_ERROR
"""
#Retrieve the POOL File GUID(s) for any final output files
self.log.info('Will search for POOL GUIDs for: %s' %(', '.join(candidateFiles.keys())))
pfnGUIDs = {}
generated = []
for fname in candidateFiles.keys():
guid = makeGuid(fname)
pfnGUIDs[fname] = guid
generated.append(fname)
pfnGUID = S_OK(pfnGUIDs)
pfnGUID['generated'] = generated
#pfnGUID = getGUID(candidateFiles.keys())
#if not pfnGUID['OK']:
# self.log.error('PoolXMLFile failed to determine POOL GUID(s) for output file list, these will be generated by \
# the DataManager',pfnGUID['Message'])
# for fileName in candidateFiles.keys():
# candidateFiles[fileName]['guid']=''
#if pfnGUID['generated']:
self.log.debug('Generated GUID(s) for the following files ', ', '.join(pfnGUID['generated']))
#else:
# self.log.info('GUIDs found for all specified POOL files: %s' %(string.join(candidateFiles.keys(),', ')))
for pfn, guid in pfnGUID['Value'].items():
candidateFiles[pfn]['GUID'] = guid
#Get all additional metadata about the file necessary for requests
final = {}
for fileName, metadata in candidateFiles.items():
fileDict = {}
fileDict['LFN'] = metadata['lfn']
fileDict['Size'] = os.path.getsize(fileName)
fileDict['Addler'] = fileAdler(fileName)
fileDict['ADLER32'] = fileAdler(fileName)
fileDict['Checksum'] = fileAdler(fileName)
fileDict['ChecksumType'] = "ADLER32"
fileDict['GUID'] = metadata['GUID']
fileDict['Status'] = "Waiting"
final[fileName] = metadata
final[fileName]['filedict'] = fileDict
final[fileName]['localpath'] = '%s/%s' % (os.getcwd(), fileName)
#Sanity check all final candidate metadata keys are present (return S_ERROR if not)
mandatoryKeys = ['GUID', 'filedict'] #filedict is used for requests (this method adds guid and filedict)
for fileName, metadata in final.items():
for key in mandatoryKeys:
if not key in metadata:
return S_ERROR('File %s has missing %s' % (fileName, key))
return S_OK(final)
def __stat( path ):
""" Issue a stat call and format it the dirac way, and add the checksum
:param self: self reference
:param path: path on the storage
:returns Successful S_OK(metadataDict) or S_ERROR
"""
try:
statInfo = os.stat( path )
metadataDict = {}
metadataDict['ModTime'] = statInfo.st_mtime
metadataDict['Size'] = statInfo.st_size
metadataDict['Mode'] = stat.S_IMODE( statInfo.st_mode )
metadataDict['Directory'] = bool( stat.S_ISDIR( statInfo.st_mode ) )
isFile = bool( stat.S_ISREG( statInfo.st_mode ) )
metadataDict['File'] = isFile
cks = ""
if isFile:
cks = fileAdler( path )
metadataDict['Checksum'] = cks if cks else ""
# FIXME: only here for compatibility with SRM until multi protocol is properly handled
metadataDict['Cached'] = 1
metadataDict['Migrated'] = 0
metadataDict['Lost'] = 0
metadataDict['Unavailable'] = 0
except OSError as ose:
return S_ERROR( str( ose ) )
return S_OK( metadataDict )
def registerFilesTree(catalog, lfn, pfn):
global processedFiles
fileDict = {}
dirlist = os.listdir(pfn)
for entry in dirlist:
entrypfn = "%s/%s" % (pfn, entry)
entrylfn = "%s/%s" % (lfn, entry)
if os.path.isdir(entrypfn):
registerFilesTree(catalog, entrylfn, entrypfn)
continue
infoDict = {}
infoDict['PFN'] = entrypfn
try:
size = os.fstat(entrypfn).ST_SIZE
infoDict['Size'] = int(size)
except Exception:
print "Setting size 0, because error getting ls -l " + entrypfn
infoDict['Size'] = 0
infoDict['SE'] = 'EISCAT-disk'
# make a UUID based on the host ID and current time
#infoDict['GUID'] = uuid.uuid1().int
infoDict['GUID'] = str(uuid.uuid1().hex)
infoDict['Checksum'] = ''
if registerChecksum:
infoDict['Checksum'] = str(fileAdler(entrypfn))
fileDict[entrylfn] = infoDict
# bulk files registration in the current level
if fileDict:
print "Registering %d files in %s directory" % (len(fileDict), pfn)
registerFiles(catalog, fileDict)
processedFiles += len(fileDict)
def __stat(path):
""" Issue a stat call and format it the dirac way, and add the checksum
:param self: self reference
:param path: path on the storage
:returns Successful S_OK(metadataDict) or S_ERROR
"""
try:
statInfo = os.stat(path)
metadataDict = {}
metadataDict['ModTime'] = statInfo.st_mtime
metadataDict['Size'] = statInfo.st_size
metadataDict['Mode'] = stat.S_IMODE(statInfo.st_mode)
metadataDict['Directory'] = bool(stat.S_ISDIR(statInfo.st_mode))
isFile = bool(stat.S_ISREG(statInfo.st_mode))
metadataDict['File'] = isFile
cks = ""
if isFile:
cks = fileAdler(path)
metadataDict['Checksum'] = cks if cks else ""
# FIXME: only here for compatibility with SRM until multi protocol is properly handled
metadataDict['Cached'] = 1
metadataDict['Migrated'] = 0
metadataDict['Lost'] = 0
metadataDict['Unavailable'] = 0
except OSError as ose:
return S_ERROR(str(ose))
return S_OK(metadataDict)
def getFileMetadata(self, candidateFiles):
"""Returns the candidate file dictionary with associated metadata.
@param candidateFiles: The input candidate files dictionary has the structure:
{'lfn':'','path':'','workflowSE':''}
This also assumes the files are in the current working directory.
@return: File Metadata
"""
# Retrieve the POOL File GUID(s) for any final output files
self.log.info("Will search for POOL GUIDs for: %s" % (string.join(candidateFiles.keys(), ", ")))
pfnGUIDs = {}
generated = []
for fname in candidateFiles.keys():
guid = makeGuid(fname)
pfnGUIDs[fname] = guid
generated.append(fname)
pfnGUID = S_OK(pfnGUIDs)
pfnGUID["generated"] = generated
# pfnGUID = getGUID(candidateFiles.keys())
# if not pfnGUID['OK']:
# self.log.error('PoolXMLFile failed to determine POOL GUID(s) for output file list, these will be generated by \
# the ReplicaManager',pfnGUID['Message'])
# for fileName in candidateFiles.keys():
# candidateFiles[fileName]['guid']=''
# if pfnGUID['generated']:
self.log.debug("Generated GUID(s) for the following files ", string.join(pfnGUID["generated"], ", "))
# else:
# self.log.info('GUIDs found for all specified POOL files: %s' %(string.join(candidateFiles.keys(),', ')))
for pfn, guid in pfnGUID["Value"].items():
candidateFiles[pfn]["guid"] = guid
# Get all additional metadata about the file necessary for requests
final = {}
for fileName, metadata in candidateFiles.items():
fileDict = {}
fileDict["LFN"] = metadata["lfn"]
fileDict["Size"] = os.path.getsize(fileName)
fileDict["Addler"] = fileAdler(fileName)
fileDict["GUID"] = metadata["guid"]
fileDict["Status"] = "Waiting"
final[fileName] = metadata
final[fileName]["filedict"] = fileDict
final[fileName]["localpath"] = "%s/%s" % (os.getcwd(), fileName)
# Sanity check all final candidate metadata keys are present (return S_ERROR if not)
mandatoryKeys = ["guid", "filedict"] # filedict is used for requests (this method adds guid and filedict)
for fileName, metadata in final.items():
for key in mandatoryKeys:
if not metadata.has_key(key):
return S_ERROR("File %s has missing %s" % (fileName, key))
return S_OK(final)
def getFileMetadata(self, candidateFiles):
"""Returns the candidate file dictionary with associated metadata.
@param candidateFiles: The input candidate files dictionary has the structure:
{'lfn':'','path':'','workflowSE':''}
This also assumes the files are in the current working directory.
@return: File Metadata
"""
#Retrieve the POOL File GUID(s) for any final output files
self.log.info('Will search GUIDs for: %s' %
(', '.join(candidateFiles.keys())))
pfnGUIDs = {}
generated = []
for fname in candidateFiles.keys():
guid = makeGuid(fname)
pfnGUIDs[fname] = guid
generated.append(fname)
pfnGUID = S_OK(pfnGUIDs)
pfnGUID['generated'] = generated
self.log.debug('Generated GUID(s) for the following files ',
', '.join(pfnGUID['generated']))
for pfn, guid in pfnGUID['Value'].items():
candidateFiles[pfn]['GUID'] = guid
#Get all additional metadata about the file necessary for requests
final = {}
for fileName, metadata in candidateFiles.items():
fileDict = {}
fileDict['LFN'] = metadata['lfn']
fileDict['Size'] = os.path.getsize(fileName)
fileDict['Addler'] = fileAdler(fileName)
fileDict['GUID'] = metadata['GUID']
fileDict['Status'] = "Waiting"
final[fileName] = metadata
final[fileName]['filedict'] = fileDict
final[fileName]['localpath'] = '%s/%s' % (os.getcwd(), fileName)
gLogger.verbose("Full file dict", str(final))
#Sanity check all final candidate metadata keys are present (return S_ERROR if not)
mandatoryKeys = [
'GUID', 'filedict'
] #filedict is used for requests (this method adds guid and filedict)
for fileName, metadata in final.items():
for key in mandatoryKeys:
if not metadata.has_key(key):
return S_ERROR('File %s has missing %s' % (fileName, key))
return S_OK(final)
def getFileMetadata(self, candidateFiles):
""" Returns the candidate file dictionary with associated metadata.
The input candidate files dictionary has the structure:
{'foo_1.txt': {'lfn': '/lhcb/MC/2010/DST/00012345/0001/foo_1.txt',
'type': 'txt',
'workflowSE': SE1},
'bar_2.py': {'lfn': '/lhcb/MC/2010/DST/00012345/0001/bar_2.py',
'type': 'py',
'workflowSE': 'SE2'},
}
this also assumes the files are in the current working directory.
"""
# Retrieve the POOL File GUID(s) for any final output files
self.log.info('Will search for POOL GUIDs for: %s' % (', '.join(candidateFiles.keys())))
pfnGUID = getGUID(candidateFiles.keys())
if not pfnGUID['OK']:
self.log.error('''PoolXMLFile failed to determine POOL GUID(s) for output file list,
these will be generated by the DataManager''', pfnGUID['Message'])
for fileName in candidateFiles.keys():
candidateFiles[fileName]['guid'] = ''
elif pfnGUID['generated']:
self.log.warn('PoolXMLFile generated GUID(s) for the following files ', ', '.join(pfnGUID['generated']))
else:
self.log.info('GUIDs found for all specified POOL files: %s' % (', '.join(candidateFiles.keys())))
for pfn, guid in pfnGUID['Value'].iteritems():
candidateFiles[pfn]['guid'] = guid
# Get all additional metadata about the file necessary for requests
final = {}
for fileName, metadata in candidateFiles.iteritems():
fileDict = {}
fileDict['LFN'] = metadata['lfn']
fileDict['Size'] = os.path.getsize(fileName)
fileDict['Checksum'] = fileAdler(fileName)
fileDict['ChecksumType'] = 'ADLER32'
fileDict['GUID'] = metadata['guid']
fileDict['Status'] = 'Waiting'
final[fileName] = metadata
final[fileName]['filedict'] = fileDict
final[fileName]['localpath'] = '%s/%s' % (os.getcwd(), fileName)
# Sanity check all final candidate metadata keys are present (return S_ERROR if not)
mandatoryKeys = ['guid', 'filedict'] # filedict is used for requests (this method adds guid and filedict)
for fileName, metadata in final.iteritems():
for key in mandatoryKeys:
if key not in metadata:
raise RuntimeError("File %s has missing %s" % (fileName, key))
return final
def getFileMetadata(self, candidateFiles):
"""Returns the candidate file dictionary with associated metadata.
@param candidateFiles: The input candidate files dictionary has the structure:
{'lfn':'','path':'','workflowSE':''}
This also assumes the files are in the current working directory.
@return: File Metadata
"""
#Retrieve the POOL File GUID(s) for any final output files
self.log.info('Will search GUIDs for: %s' %(', '.join(candidateFiles.keys())))
pfnGUIDs = {}
generated = []
for fname in candidateFiles.keys():
guid = makeGuid(fname)
pfnGUIDs[fname] = guid
generated.append(fname)
pfnGUID = S_OK(pfnGUIDs)
pfnGUID['generated'] = generated
self.log.debug('Generated GUID(s) for the following files ', ', '.join(pfnGUID['generated']))
for pfn, guid in pfnGUID['Value'].items():
candidateFiles[pfn]['GUID'] = guid
#Get all additional metadata about the file necessary for requests
final = {}
for fileName, metadata in candidateFiles.items():
fileDict = {}
fileDict['LFN'] = metadata['lfn']
fileDict['Size'] = os.path.getsize(fileName)
fileDict['Addler'] = fileAdler(fileName)
fileDict['GUID'] = metadata['GUID']
fileDict['Status'] = "Waiting"
final[fileName] = metadata
final[fileName]['filedict'] = fileDict
final[fileName]['localpath'] = '%s/%s' % (os.getcwd(), fileName)
gLogger.verbose("Full file dict", str(final))
#Sanity check all final candidate metadata keys are present (return S_ERROR if not)
mandatoryKeys = ['GUID', 'filedict'] #filedict is used for requests (this method adds guid and filedict)
for fileName, metadata in final.items():
for key in mandatoryKeys:
if not metadata.has_key(key):
return S_ERROR('File %s has missing %s' % (fileName, key))
return S_OK(final)
def main():
dm = DataManager()
fileTupleBuffer = []
counter = 0
for f in files:
counter += 1
if not f.startswith('/cefs'):
gLogger.error('File must be under "/cefs"')
return 1
lfn = '/cepc/lustre-ro' + f
result = fcc.isFile(lfn)
if result['OK'] and lfn in result['Value']['Successful'] and result[
'Value']['Successful'][lfn]:
continue
size = os.path.getsize(f)
adler32 = fileAdler(f)
guid = makeGuid()
fileTuple = (lfn, f, size, _se, guid, adler32)
fileTupleBuffer.append(fileTuple)
gLogger.debug('Register to lfn: %s' % lfn)
gLogger.debug('fileTuple: %s' % (fileTuple, ))
if len(fileTupleBuffer) >= _bufferSize:
result = dm.registerFile(fileTupleBuffer)
print('register result', result)
if not result['OK']:
gLogger.error('Register file failed')
return 1
del fileTupleBuffer[:]
gLogger.debug('%s files registered' % counter)
if fileTupleBuffer:
result = dm.registerFile(fileTupleBuffer)
print('register result', result)
if not result['OK']:
gLogger.error('Register file failed')
return 1
del fileTupleBuffer[:]
gLogger.info('Totally %s files registered' % counter)
return 0
def files(self, userName, userGroup):
"""get list of files in user domain"""
files = []
for i in range(10):
fname = "/tmp/testUserFile-%s" % i
if userGroup == "dteam_user":
lfn = "/lhcb/user/%s/%s/%s" % (userName[0], userName, fname.split("/")[-1])
else:
lfn = "/lhcb/certification/test/rmsdms/%s" % fname.split("/")[-1]
fh = open(fname, "w+")
for i in range(100):
fh.write(str(random.randint(0, i)))
fh.close()
size = os.stat(fname).st_size
checksum = fileAdler(fname)
guid = makeGuid(fname)
files.append((fname, lfn, size, checksum, guid))
return files
def files( self, userName, userGroup ):
""" get list of files in user domain """
files = []
for i in range( 10 ):
fname = "/tmp/testUserFile-%s" % i
if userGroup == "lhcb_user":
lfn = "/lhcb/user/%s/%s/%s" % ( userName[0], userName, fname.split( "/" )[-1] )
else:
lfn = "/lhcb/certification/test/rmsdms/%s" % fname.split( "/" )[-1]
fh = open( fname, "w+" )
for i in range( 100 ):
fh.write( str( random.randint( 0, i ) ) )
fh.close()
size = os.stat( fname ).st_size
checksum = fileAdler( fname )
guid = makeGuid( fname )
files.append( ( fname, lfn, size, checksum, guid ) )
return files
def main():
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(["File: File Name"])
_, files = Script.parseCommandLine(ignoreErrors=False)
exitCode = 0
import DIRAC
from DIRAC.Core.Utilities.Adler import fileAdler
for fa in files:
adler = fileAdler(fa)
if adler:
print(fa.rjust(100), adler.ljust(10)) # pylint: disable=no-member
else:
print("ERROR %s: Failed to get adler" % fa)
exitCode = 2
DIRAC.exit(exitCode)
def main():
Script.parseCommandLine(ignoreErrors=False)
files = Script.getPositionalArgs()
if len(files) == 0:
Script.showHelp()
exitCode = 0
import DIRAC
from DIRAC.Core.Utilities.Adler import fileAdler
for fa in files:
adler = fileAdler(fa)
if adler:
print(fa.rjust(100), adler.ljust(10)) # pylint: disable=no-member
else:
print('ERROR %s: Failed to get adler' % fa)
exitCode = 2
DIRAC.exit(exitCode)
def _direct_putFile(self, urls, sourceSize=0):
"""Upload a local file.
..warning:: no 3rd party copy possible
:param urls: dictionary { urls : localFile }
:param sourceSize: size of the file in byte. Mandatory for third party copy (WHY ???)
Also, this parameter makes it essentially a non bulk operation for
third party copy, unless all files have the same size...
:returns: * Successful dict: { path : size }
* Failed dict: { path : error message }
* S_ERROR in case of argument problems
"""
log = LOG.getSubLogger("putFile")
# the @_extractKeyFromS3Path transformed URL into keys
keys = urls
failed = {}
successful = {}
for dest_key, src_file in keys.items():
try:
cks = fileAdler(src_file)
if not cks:
log.warn("Cannot get ADLER32 checksum for %s" % src_file)
with open(src_file, "rb") as src_fd:
self.s3_client.put_object(Body=src_fd,
Bucket=self.bucketName,
Key=dest_key,
Metadata={"Checksum": cks})
successful[dest_key] = os.path.getsize(src_file)
except Exception as e:
failed[dest_key] = repr(e)
return S_OK({"Failed": failed, "Successful": successful})
def __getFileStat(path):
""" Get the file stat information
"""
resultDict = {}
try:
statTuple = os.stat(path)
except OSError as x:
if str(x).find('No such file') >= 0:
resultDict['Exists'] = False
return S_OK(resultDict)
return S_ERROR('Failed to get metadata for %s' % path)
resultDict['Exists'] = True
mode = statTuple[stat.ST_MODE]
resultDict['Type'] = "File"
resultDict['File'] = True
resultDict['Directory'] = False
if stat.S_ISDIR(mode):
resultDict['Type'] = "Directory"
resultDict['File'] = False
resultDict['Directory'] = True
resultDict['Size'] = statTuple[stat.ST_SIZE]
resultDict['TimeStamps'] = (statTuple[stat.ST_ATIME],
statTuple[stat.ST_MTIME],
statTuple[stat.ST_CTIME])
resultDict['Cached'] = 1
resultDict['Migrated'] = 0
resultDict['Lost'] = 0
resultDict['Unavailable'] = 0
resultDict['Mode'] = stat.S_IMODE(mode)
if resultDict['File']:
cks = fileAdler(path)
resultDict['Checksum'] = cks
resultDict = StorageBase._addCommonMetadata(resultDict)
return S_OK(resultDict)
def __getFileStat(path):
"""Get the file stat information"""
resultDict = {}
try:
statTuple = os.stat(path)
except OSError as x:
if str(x).find("No such file") >= 0:
resultDict["Exists"] = False
return S_OK(resultDict)
return S_ERROR("Failed to get metadata for %s" % path)
resultDict["Exists"] = True
mode = statTuple[stat.ST_MODE]
resultDict["Type"] = "File"
resultDict["File"] = True
resultDict["Directory"] = False
if stat.S_ISDIR(mode):
resultDict["Type"] = "Directory"
resultDict["File"] = False
resultDict["Directory"] = True
resultDict["Size"] = statTuple[stat.ST_SIZE]
resultDict["TimeStamps"] = (statTuple[stat.ST_ATIME],
statTuple[stat.ST_MTIME],
statTuple[stat.ST_CTIME])
resultDict["Cached"] = 1
resultDict["Migrated"] = 0
resultDict["Lost"] = 0
resultDict["Unavailable"] = 0
resultDict["Mode"] = stat.S_IMODE(mode)
if resultDict["File"]:
cks = fileAdler(path)
resultDict["Checksum"] = cks
resultDict = StorageBase._addCommonMetadata(resultDict)
return S_OK(resultDict)
def __getFileStat(path):
""" Get the file stat information
"""
resultDict = {}
try:
statTuple = os.stat(path)
except OSError as x:
if str(x).find('No such file') >= 0:
resultDict['Exists'] = False
return S_OK(resultDict)
return S_ERROR('Failed to get metadata for %s' % path)
resultDict['Exists'] = True
mode = statTuple[stat.ST_MODE]
resultDict['Type'] = "File"
resultDict['File'] = True
resultDict['Directory'] = False
if stat.S_ISDIR(mode):
resultDict['Type'] = "Directory"
resultDict['File'] = False
resultDict['Directory'] = True
resultDict['Size'] = statTuple[stat.ST_SIZE]
resultDict['TimeStamps'] = (statTuple[stat.ST_ATIME], statTuple[stat.ST_MTIME], statTuple[stat.ST_CTIME])
resultDict['Cached'] = 1
resultDict['Migrated'] = 0
resultDict['Lost'] = 0
resultDict['Unavailable'] = 0
resultDict['Mode'] = stat.S_IMODE(mode)
if resultDict['File']:
cks = fileAdler(path)
resultDict['Checksum'] = cks
resultDict = StorageBase._addCommonMetadata(resultDict)
return S_OK(resultDict)
if lfn in lfnQuery:
if counter % 1000 == 0:
gLogger.notice('Skip file in query counter: %s' % counter)
continue
if existCheck:
result = fcc.isFile(lfn)
if result['OK'] and lfn in result['Value'][
'Successful'] and result['Value']['Successful'][lfn]:
if counter % 1000 == 0:
gLogger.notice('Skip file existed counter: %s' % counter)
continue
size = os.path.getsize(fullFn)
adler32 = fileAdler(fullFn)
guid = makeGuid()
fileTuple = (lfn, fullFn, size, toSE, guid, adler32)
fileTupleBuffer.append(fileTuple)
gLogger.debug('Register to lfn: %s' % lfn)
gLogger.debug('fileTuple: %s' % (fileTuple, ))
if len(fileTupleBuffer) >= bufferSize:
result = dm.registerFile(fileTupleBuffer)
if not result['OK']:
gLogger.error('Can not register %s' % fullFn)
exit(1)
del fileTupleBuffer[:]
gLogger.notice('%s files registered' % counter)
if fileTupleBuffer:
def testWorkflow(self):
""" This perform a complete workflow puting, removing, stating files and directories
"""
putDir = {
os.path.join(DESTINATION_PATH, 'Workflow/FolderA'):
os.path.join(self.LOCAL_PATH, 'Workflow/FolderA'),
os.path.join(DESTINATION_PATH, 'Workflow/FolderB'):
os.path.join(self.LOCAL_PATH, 'Workflow/FolderB')
}
createDir = [
os.path.join(DESTINATION_PATH, 'Workflow/FolderA/FolderAA'),
os.path.join(DESTINATION_PATH, 'Workflow/FolderA/FolderABA'),
os.path.join(DESTINATION_PATH, 'Workflow/FolderA/FolderAAB')
]
putFile = {
os.path.join(DESTINATION_PATH, 'Workflow/FolderA/File1'):
os.path.join(self.LOCAL_PATH, 'Workflow/File1'),
os.path.join(DESTINATION_PATH, 'Workflow/FolderAA/File1'):
os.path.join(self.LOCAL_PATH, 'Workflow/File1'),
os.path.join(DESTINATION_PATH, 'Workflow/FolderBB/File2'):
os.path.join(self.LOCAL_PATH, 'Workflow/File2'),
os.path.join(DESTINATION_PATH, 'Workflow/FolderB/File2'):
os.path.join(self.LOCAL_PATH, 'Workflow/File2'),
os.path.join(DESTINATION_PATH, 'Workflow/File3'):
os.path.join(self.LOCAL_PATH, 'Workflow/File3')
}
isFile = {
os.path.join(DESTINATION_PATH, 'Workflow/FolderA/File1'):
os.path.join(self.LOCAL_PATH, 'Workflow/File1'),
os.path.join(DESTINATION_PATH, 'Workflow/FolderB/FileB'):
os.path.join(self.LOCAL_PATH, 'Workflow/FolderB/FileB'),
}
listDir = [
os.path.join(DESTINATION_PATH, 'Workflow'),
os.path.join(DESTINATION_PATH, 'Workflow/FolderA'),
os.path.join(DESTINATION_PATH, 'Workflow/FolderB')
]
getDir = [
os.path.join(DESTINATION_PATH, 'Workflow/FolderA'),
os.path.join(DESTINATION_PATH, 'Workflow/FolderB')
]
removeFile = [os.path.join(DESTINATION_PATH, 'Workflow/FolderA/File1')]
rmdir = [os.path.join(DESTINATION_PATH, 'Workflow')]
##### Computing local adler and size #####
fileAdlers = {}
fileSizes = {}
for lfn, localFn in isFile.iteritems():
fileAdlers[lfn] = fileAdler(localFn)
fileSizes[lfn] = getSize(localFn)
########## uploading directory #############
res = self.writeSE.putDirectory(putDir)
self.assertEqual(res['OK'], True)
# time.sleep(5)
res = self.readSE.listDirectory(listDir)
self.assertEqual(
any(
os.path.join(DESTINATION_PATH, 'Workflow/FolderA/FileA') in
dictKey for dictKey in res['Value']['Successful'][os.path.join(
DESTINATION_PATH, 'Workflow/FolderA')]['Files'].keys()),
True)
self.assertEqual(
any(
os.path.join(DESTINATION_PATH, 'Workflow/FolderB/FileB') in
dictKey for dictKey in res['Value']['Successful'][os.path.join(
DESTINATION_PATH, 'Workflow/FolderB')]['Files'].keys()),
True)
########## createDir #############
res = self.writeSE.createDirectory(createDir)
self.assertEqual(res['OK'], True)
res = res['Value']
self.assertEqual(res['Successful'][createDir[0]], True)
self.assertEqual(res['Successful'][createDir[1]], True)
self.assertEqual(res['Successful'][createDir[2]], True)
######## putFile ########
res = self.writeSE.putFile(putFile)
self.assertEqual(res['OK'], True)
# time.sleep(5)
res = self.readSE.isFile(isFile)
self.assertEqual(res['OK'], True)
self.assertTrue(
all([x for x in res['Value']['Successful'].itervalues()]))
# self.assertEqual( res['Value']['Successful'][isFile[0]], True )
# self.assertEqual( res['Value']['Successful'][isFile[1]], True )
######## getMetadata ###########
res = self.readSE.getFileMetadata(isFile)
self.assertEqual(res['OK'], True)
res = res['Value']['Successful']
self.assertEqual(
any(path in resKey for path in isFile for resKey in res.keys()),
True)
# Checking that the checksums and sizes are correct
for lfn in isFile:
self.assertEqual(res[lfn]['Checksum'], fileAdlers[lfn])
self.assertEqual(res[lfn]['Size'], fileSizes[lfn])
####### getDirectory ######
res = self.readSE.getDirectory(getDir,
os.path.join(self.LOCAL_PATH, 'getDir'))
self.assertEqual(res['OK'], True)
res = res['Value']
self.assertEqual(
any(getDir[0] in dictKey for dictKey in res['Successful']), True)
self.assertEqual(
any(getDir[1] in dictKey for dictKey in res['Successful']), True)
###### removeFile ##########
res = self.writeSE.removeFile(removeFile)
self.assertEqual(res['OK'], True)
res = self.readSE.exists(removeFile)
self.assertEqual(res['OK'], True)
self.assertEqual(res['Value']['Successful'][removeFile[0]], False)
###### remove non existing file #####
res = self.writeSE.removeFile(removeFile)
self.assertEqual(res['OK'], True)
res = self.readSE.exists(removeFile)
self.assertEqual(res['OK'], True)
self.assertEqual(res['Value']['Successful'][removeFile[0]], False)
########### removing directory ###########
res = self.writeSE.removeDirectory(rmdir, True)
res = self.readSE.exists(rmdir)
self.assertEqual(res['OK'], True)
self.assertEqual(res['Value']['Successful'][rmdir[0]], False)
def setuptest(request):
global local_path, putDir, createDir, putFile, isFile, listDir,\
getDir, getFile, rmDir, removeFile, se, filesInFolderAandB, fileAdlers, fileSizes
local_path = tempfile.mkdtemp()
# create the local structure
workPath = os.path.join(local_path, 'Workflow')
os.mkdir(workPath)
os.mkdir(os.path.join(workPath, 'FolderA'))
with open(os.path.join(workPath, 'FolderA', 'FileA'), 'w') as f:
f.write(_mul('FileA'))
os.mkdir(os.path.join(workPath, 'FolderA', 'FolderAA'))
with open(os.path.join(workPath, 'FolderA', 'FolderAA', 'FileAA'),
'w') as f:
f.write(_mul('FileAA'))
os.mkdir(os.path.join(workPath, 'FolderB'))
with open(os.path.join(workPath, 'FolderB', 'FileB'), 'w') as f:
f.write(_mul('FileB'))
for fn in ["File1", "File2", "File3"]:
with open(os.path.join(workPath, fn), 'w') as f:
f.write(_mul(fn))
se = StorageElement(STORAGE_NAME)
putDir = {
os.path.join(DESTINATION_PATH, 'Workflow/FolderA'):
os.path.join(local_path, 'Workflow/FolderA'),
os.path.join(DESTINATION_PATH, 'Workflow/FolderB'):
os.path.join(local_path, 'Workflow/FolderB')
}
createDir = [
os.path.join(DESTINATION_PATH, 'Workflow/FolderA/FolderAA'),
os.path.join(DESTINATION_PATH, 'Workflow/FolderA/FolderABA'),
os.path.join(DESTINATION_PATH, 'Workflow/FolderA/FolderAAB')
]
putFile = {
os.path.join(DESTINATION_PATH, 'Workflow/FolderA/File1'):
os.path.join(local_path, 'Workflow/File1'),
os.path.join(DESTINATION_PATH, 'Workflow/FolderAA/File1'):
os.path.join(local_path, 'Workflow/File1'),
os.path.join(DESTINATION_PATH, 'Workflow/FolderBB/File2'):
os.path.join(local_path, 'Workflow/File2'),
os.path.join(DESTINATION_PATH, 'Workflow/FolderB/File2'):
os.path.join(local_path, 'Workflow/File2'),
os.path.join(DESTINATION_PATH, 'Workflow/File3'):
os.path.join(local_path, 'Workflow/File3')
}
isFile = putFile.keys()
listDir = [
os.path.join(DESTINATION_PATH, 'Workflow'),
os.path.join(DESTINATION_PATH, 'Workflow/FolderA'),
os.path.join(DESTINATION_PATH, 'Workflow/FolderB')
]
getDir = [
os.path.join(DESTINATION_PATH, 'Workflow/FolderA'),
os.path.join(DESTINATION_PATH, 'Workflow/FolderB')
]
removeFile = [os.path.join(DESTINATION_PATH, 'Workflow/FolderA/File1')]
rmdir = [os.path.join(DESTINATION_PATH, 'Workflow')]
# This list is used to check for existance of files
# after uploading the directory: they should NOT exist.
# Uploading a directory does not work.
filesInFolderAandB = []
for dirName in ('Workflow/FolderA', 'Workflow/FolderB'):
for root, _dirs, files in os.walk(os.path.join(local_path, dirName)):
for fn in files:
filesInFolderAandB.append(
os.path.join(DESTINATION_PATH,
root.replace(local_path, '').strip('/'), fn))
filesInFolderAandB = dict.fromkeys(filesInFolderAandB, False)
fileAdlers = {}
fileSizes = {}
for lfn, localFn in putFile.iteritems():
fileAdlers[lfn] = fileAdler(localFn)
fileSizes[lfn] = getSize(localFn)
clearDirectory(se, local_path, DESTINATION_PATH)
def teardown():
print("Cleaning local test")
shutil.rmtree(local_path)
clearDirectory(se, local_path, DESTINATION_PATH)
request.addfinalizer(teardown)
return local_path, random.randint(0, 100) # provide the fixture value
########################################################################
"""
Calculate alder32 of the supplied file
"""
__RCSID__ = "$Id$"
import DIRAC
from DIRAC.Core.Utilities.Adler import fileAdler
from DIRAC.Core.Base import Script
Script.setUsageMessage('\n'.join([
__doc__.split('\n')[1], 'Usage:',
' %s [option|cfgfile] ... File ...' % Script.scriptName, 'Arguments:',
' File: File Name'
]))
Script.parseCommandLine(ignoreErrors=False)
files = Script.getPositionalArgs()
if len(files) == 0:
Script.showHelp()
exitCode = 0
for fa in files:
adler = fileAdler(fa)
if adler:
print fa.rjust(100), adler.ljust(10)
else:
print 'ERROR %s: Failed to get adler' % fa
exitCode = 2
DIRAC.exit(exitCode)
from DIRAC.RequestManagementSystem.Client.Request import Request
from DIRAC.RequestManagementSystem.Client.Operation import Operation
from DIRAC.RequestManagementSystem.Client.File import File
from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient
from DIRAC.Core.Utilities.Adler import fileAdler
if not os.path.exists( PFN ):
gLogger.error( "%s does not exist" % PFN )
DIRAC.exit( -1 )
if not os.path.isfile( PFN ):
gLogger.error( "%s is not a file" % PFN )
DIRAC.exit( -1 )
PFN = os.path.abspath( PFN )
size = os.path.getsize( PFN )
adler32 = fileAdler( PFN )
request = Request()
request.RequestName = requestName
putAndRegister = Operation()
purAndRegister.Type = "PutAndRegister"
purAndRegister.TargetSE = targetSE
opFile = File()
opFile.LFN = LFN
opFile.PFN = PFN
opFile.Size = size
opFile.Checksum = adler32
opFile.ChecksumType = "ADLER32"
putAndRegister.addFile( opFile )
########################################################################
"""
Calculate alder32 of the supplied file
"""
__RCSID__ = "$Id$"
import DIRAC
from DIRAC.Core.Utilities.Adler import fileAdler
from DIRAC.Core.Base import Script
Script.setUsageMessage('\n'.join([
__doc__.split('\n')[1], 'Usage:',
' %s [option|cfgfile] ... File ...' % Script.scriptName, 'Arguments:',
' File: File Name'
]))
Script.parseCommandLine(ignoreErrors=False)
files = Script.getPositionalArgs()
if len(files) == 0:
Script.showHelp()
exitCode = 0
for file in files:
adler = fileAdler(file)
if adler:
print file.rjust(100), adler.ljust(10)
else:
print 'ERROR %s: Failed to get adler' % file
exitCode = 2
DIRAC.exit(exitCode)
from DIRAC.RequestManagementSystem.Client.Request import Request
from DIRAC.RequestManagementSystem.Client.Operation import Operation
from DIRAC.RequestManagementSystem.Client.File import File
from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient
from DIRAC.Core.Utilities.Adler import fileAdler
if not os.path.exists( PFN ):
gLogger.error( "%s does not exist" % PFN )
DIRAC.exit( -1 )
if not os.path.isfile( PFN ):
gLogger.error( "%s is not a file" % PFN )
DIRAC.exit( -1 )
PFN = os.path.abspath( PFN )
size = os.path.getsize( PFN )
adler32 = fileAdler( PFN )
request = Request()
request.RequestName = requestName
putAndRegister = Operation()
putAndRegister.Type = "PutAndRegister"
putAndRegister.TargetSE = targetSE
opFile = File()
opFile.LFN = LFN
opFile.PFN = PFN
opFile.Size = size
opFile.Checksum = adler32
opFile.ChecksumType = "ADLER32"
putAndRegister.addFile( opFile )
request.addOperation( putAndRegister )
########################################################################
"""
Calculate alder32 of the supplied file
"""
__RCSID__ = "$Id$"
import DIRAC
from DIRAC.Core.Utilities.Adler import fileAdler
from DIRAC.Core.Base import Script
Script.setUsageMessage( '\n'.join( [ __doc__.split( '\n' )[1],
'Usage:',
' %s [option|cfgfile] ... File ...' % Script.scriptName,
'Arguments:',
' File: File Name' ] ) )
Script.parseCommandLine( ignoreErrors = False )
files = Script.getPositionalArgs()
if len( files ) == 0:
Script.showHelp()
exitCode = 0
for fa in files:
adler = fileAdler( fa )
if adler:
print fa.rjust( 100 ), adler.ljust( 10 )
else:
print 'ERROR %s: Failed to get adler' % fa
exitCode = 2
DIRAC.exit( exitCode )
########################################################################
"""
Calculate alder32 of the supplied file
"""
__RCSID__ = "2b9038b (2010-12-15 07:24:22 +0000) Ricardo Graciani <*****@*****.**>"
import DIRAC
from DIRAC.Core.Utilities.Adler import fileAdler
from DIRAC.Core.Base import Script
Script.setUsageMessage( '\n'.join( [ __doc__.split( '\n' )[1],
'Usage:',
' %s [option|cfgfile] ... File ...' % Script.scriptName,
'Arguments:',
' File: File Name' ] ) )
Script.parseCommandLine( ignoreErrors = False )
files = Script.getPositionalArgs()
if len( files ) == 0:
Script.showHelp()
exitCode = 0
for file in files:
adler = fileAdler( file )
if adler:
print file.rjust( 100 ), adler.ljust( 10 )
else:
print 'ERROR %s: Failed to get adler' % file
exitCode = 2
DIRAC.exit( exitCode )
def testWorkflow(self):
""" This perform a complete workflow puting, removing, stating files and directories
"""
putDir = {os.path.join(DESTINATION_PATH,
'Workflow/FolderA'): os.path.join(self.LOCAL_PATH,
'Workflow/FolderA'),
os.path.join(DESTINATION_PATH,
'Workflow/FolderB'): os.path.join(self.LOCAL_PATH,
'Workflow/FolderB')}
createDir = [os.path.join(DESTINATION_PATH, 'Workflow/FolderA/FolderAA'),
os.path.join(DESTINATION_PATH, 'Workflow/FolderA/FolderABA'),
os.path.join(DESTINATION_PATH, 'Workflow/FolderA/FolderAAB')
]
putFile = {os.path.join(DESTINATION_PATH,
'Workflow/FolderA/File1'): os.path.join(self.LOCAL_PATH,
'Workflow/File1'),
os.path.join(DESTINATION_PATH,
'Workflow/FolderAA/File1'): os.path.join(self.LOCAL_PATH,
'Workflow/File1'),
os.path.join(DESTINATION_PATH,
'Workflow/FolderBB/File2'): os.path.join(self.LOCAL_PATH,
'Workflow/File2'),
os.path.join(DESTINATION_PATH,
'Workflow/FolderB/File2'): os.path.join(self.LOCAL_PATH,
'Workflow/File2'),
os.path.join(DESTINATION_PATH,
'Workflow/File3'): os.path.join(self.LOCAL_PATH,
'Workflow/File3')}
isFile = {os.path.join(DESTINATION_PATH,
'Workflow/FolderA/File1'): os.path.join(self.LOCAL_PATH,
'Workflow/File1'),
os.path.join(DESTINATION_PATH,
'Workflow/FolderB/FileB'): os.path.join(self.LOCAL_PATH,
'Workflow/FolderB/FileB'),
}
listDir = [os.path.join(DESTINATION_PATH, 'Workflow'),
os.path.join(DESTINATION_PATH, 'Workflow/FolderA'),
os.path.join(DESTINATION_PATH, 'Workflow/FolderB')
]
getDir = [os.path.join(DESTINATION_PATH, 'Workflow/FolderA'),
os.path.join(DESTINATION_PATH, 'Workflow/FolderB')
]
removeFile = [os.path.join(DESTINATION_PATH, 'Workflow/FolderA/File1')]
rmdir = [os.path.join(DESTINATION_PATH, 'Workflow')]
##### Computing local adler and size #####
fileAdlers = {}
fileSizes = {}
for lfn, localFn in isFile.iteritems():
fileAdlers[lfn] = fileAdler(localFn)
fileSizes[lfn] = getSize(localFn)
########## uploading directory #############
res = self.writeSE.putDirectory(putDir)
self.assertEqual(res['OK'], True)
# time.sleep(5)
res = self.readSE.listDirectory(listDir)
self.assertEqual(any(os.path.join(DESTINATION_PATH, 'Workflow/FolderA/FileA') in dictKey for dictKey in
res['Value']['Successful'][os.path.join(DESTINATION_PATH, 'Workflow/FolderA')]['Files'].keys()), True)
self.assertEqual(any(os.path.join(DESTINATION_PATH, 'Workflow/FolderB/FileB') in dictKey for dictKey in
res['Value']['Successful'][os.path.join(DESTINATION_PATH, 'Workflow/FolderB')]['Files'].keys()), True)
########## createDir #############
res = self.writeSE.createDirectory(createDir)
self.assertEqual(res['OK'], True)
res = res['Value']
self.assertEqual(res['Successful'][createDir[0]], True)
self.assertEqual(res['Successful'][createDir[1]], True)
self.assertEqual(res['Successful'][createDir[2]], True)
######## putFile ########
res = self.writeSE.putFile(putFile)
self.assertEqual(res['OK'], True)
# time.sleep(5)
res = self.readSE.isFile(isFile)
self.assertEqual(res['OK'], True)
self.assertTrue(all([x for x in res['Value']['Successful'].itervalues()]))
# self.assertEqual( res['Value']['Successful'][isFile[0]], True )
# self.assertEqual( res['Value']['Successful'][isFile[1]], True )
######## getMetadata ###########
res = self.readSE.getFileMetadata(isFile)
self.assertEqual(res['OK'], True)
res = res['Value']['Successful']
self.assertEqual(any(path in resKey for path in isFile for resKey in res.keys()), True)
# Checking that the checksums and sizes are correct
for lfn in isFile:
self.assertEqual(res[lfn]['Checksum'], fileAdlers[lfn])
self.assertEqual(res[lfn]['Size'], fileSizes[lfn])
####### getDirectory ######
res = self.readSE.getDirectory(getDir, os.path.join(self.LOCAL_PATH, 'getDir'))
self.assertEqual(res['OK'], True)
res = res['Value']
self.assertEqual(any(getDir[0] in dictKey for dictKey in res['Successful']), True)
self.assertEqual(any(getDir[1] in dictKey for dictKey in res['Successful']), True)
###### removeFile ##########
res = self.writeSE.removeFile(removeFile)
self.assertEqual(res['OK'], True)
res = self.readSE.exists(removeFile)
self.assertEqual(res['OK'], True)
self.assertEqual(res['Value']['Successful'][removeFile[0]], False)
###### remove non existing file #####
res = self.writeSE.removeFile(removeFile)
self.assertEqual(res['OK'], True)
res = self.readSE.exists(removeFile)
self.assertEqual(res['OK'], True)
self.assertEqual(res['Value']['Successful'][removeFile[0]], False)
########### removing directory ###########
res = self.writeSE.removeDirectory(rmdir, True)
res = self.readSE.exists(rmdir)
self.assertEqual(res['OK'], True)
self.assertEqual(res['Value']['Successful'][rmdir[0]], False)
def main():
from DIRAC.Core.Base.Script import parseCommandLine
parseCommandLine()
import DIRAC
from DIRAC import gLogger
args = Script.getPositionalArgs()
requestName = None
LFN = None
PFN = None
targetSE = None
if len(args) != 4:
Script.showHelp()
else:
requestName = args[0]
LFN = args[1]
PFN = args[2]
targetSE = args[3]
if not os.path.isabs(LFN):
gLogger.error("LFN should be absolute path!!!")
DIRAC.exit(-1)
gLogger.info("will create request '%s' with 'PutAndRegister' "
"operation using %s pfn and %s target SE" %
(requestName, PFN, targetSE))
from DIRAC.RequestManagementSystem.Client.Request import Request
from DIRAC.RequestManagementSystem.Client.Operation import Operation
from DIRAC.RequestManagementSystem.Client.File import File
from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient
from DIRAC.Core.Utilities.Adler import fileAdler
if not os.path.exists(PFN):
gLogger.error("%s does not exist" % PFN)
DIRAC.exit(-1)
if not os.path.isfile(PFN):
gLogger.error("%s is not a file" % PFN)
DIRAC.exit(-1)
PFN = os.path.abspath(PFN)
size = os.path.getsize(PFN)
adler32 = fileAdler(PFN)
request = Request()
request.RequestName = requestName
putAndRegister = Operation()
putAndRegister.Type = "PutAndRegister"
putAndRegister.TargetSE = targetSE
opFile = File()
opFile.LFN = LFN
opFile.PFN = PFN
opFile.Size = size
opFile.Checksum = adler32
opFile.ChecksumType = "ADLER32"
putAndRegister.addFile(opFile)
request.addOperation(putAndRegister)
reqClient = ReqClient()
putRequest = reqClient.putRequest(request)
if not putRequest["OK"]:
gLogger.error("unable to put request '%s': %s" %
(requestName, putRequest["Message"]))
DIRAC.exit(-1)
gLogger.always("Request '%s' has been put to ReqDB for execution." %
requestName)
gLogger.always(
"You can monitor its status using command: 'dirac-rms-request %s'" %
requestName)
DIRAC.exit(0)
def main():
# Registering arguments will automatically add their description to the help menu
Script.registerArgument("requestName: a request name")
Script.registerArgument("LFN: logical file name")
Script.registerArgument("localFile: local file you want to put")
Script.registerArgument("targetSE: target SE")
Script.parseCommandLine()
import DIRAC
from DIRAC import gLogger
# parseCommandLine show help when mandatory arguments are not specified or incorrect argument
requestName, LFN, PFN, targetSE = Script.getPositionalArgs(group=True)
if not os.path.isabs(LFN):
gLogger.error("LFN should be absolute path!!!")
DIRAC.exit(-1)
gLogger.info("will create request '%s' with 'PutAndRegister' "
"operation using %s pfn and %s target SE" %
(requestName, PFN, targetSE))
from DIRAC.RequestManagementSystem.Client.Request import Request
from DIRAC.RequestManagementSystem.Client.Operation import Operation
from DIRAC.RequestManagementSystem.Client.File import File
from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient
from DIRAC.Core.Utilities.Adler import fileAdler
if not os.path.exists(PFN):
gLogger.error("%s does not exist" % PFN)
DIRAC.exit(-1)
if not os.path.isfile(PFN):
gLogger.error("%s is not a file" % PFN)
DIRAC.exit(-1)
PFN = os.path.abspath(PFN)
size = os.path.getsize(PFN)
adler32 = fileAdler(PFN)
request = Request()
request.RequestName = requestName
putAndRegister = Operation()
putAndRegister.Type = "PutAndRegister"
putAndRegister.TargetSE = targetSE
opFile = File()
opFile.LFN = LFN
opFile.PFN = PFN
opFile.Size = size
opFile.Checksum = adler32
opFile.ChecksumType = "ADLER32"
putAndRegister.addFile(opFile)
request.addOperation(putAndRegister)
reqClient = ReqClient()
putRequest = reqClient.putRequest(request)
if not putRequest["OK"]:
gLogger.error("unable to put request '%s': %s" %
(requestName, putRequest["Message"]))
DIRAC.exit(-1)
gLogger.always("Request '%s' has been put to ReqDB for execution." %
requestName)
gLogger.always(
"You can monitor its status using command: 'dirac-rms-request %s'" %
requestName)
DIRAC.exit(0)
def setuptest(request):
global local_path, putDir, createDir, putFile, isFile, listDir,\
getDir, getFile, rmDir, removeFile, se, filesInFolderAandB, fileAdlers, fileSizes
local_path = tempfile.mkdtemp()
# create the local structure
workPath = os.path.join(local_path, 'Workflow')
os.mkdir(workPath)
os.mkdir(os.path.join(workPath, 'FolderA'))
with open(os.path.join(workPath, 'FolderA', 'FileA'), 'w') as f:
f.write(_mul('FileA'))
os.mkdir(os.path.join(workPath, 'FolderA', 'FolderAA'))
with open(os.path.join(workPath, 'FolderA', 'FolderAA', 'FileAA'), 'w') as f:
f.write(_mul('FileAA'))
os.mkdir(os.path.join(workPath, 'FolderB'))
with open(os.path.join(workPath, 'FolderB', 'FileB'), 'w') as f:
f.write(_mul('FileB'))
for fn in ["File1", "File2", "File3"]:
with open(os.path.join(workPath, fn), 'w') as f:
f.write(_mul(fn))
se = StorageElement(STORAGE_NAME)
putDir = {os.path.join(DESTINATION_PATH,
'Workflow/FolderA'): os.path.join(local_path,
'Workflow/FolderA'),
os.path.join(DESTINATION_PATH,
'Workflow/FolderB'): os.path.join(local_path,
'Workflow/FolderB')}
createDir = [os.path.join(DESTINATION_PATH, 'Workflow/FolderA/FolderAA'),
os.path.join(DESTINATION_PATH, 'Workflow/FolderA/FolderABA'),
os.path.join(DESTINATION_PATH, 'Workflow/FolderA/FolderAAB')
]
putFile = {os.path.join(DESTINATION_PATH,
'Workflow/FolderA/File1'): os.path.join(local_path,
'Workflow/File1'),
os.path.join(DESTINATION_PATH,
'Workflow/FolderAA/File1'): os.path.join(local_path,
'Workflow/File1'),
os.path.join(DESTINATION_PATH,
'Workflow/FolderBB/File2'): os.path.join(local_path,
'Workflow/File2'),
os.path.join(DESTINATION_PATH,
'Workflow/FolderB/File2'): os.path.join(local_path,
'Workflow/File2'),
os.path.join(DESTINATION_PATH,
'Workflow/File3'): os.path.join(local_path,
'Workflow/File3')}
isFile = putFile.keys()
listDir = [os.path.join(DESTINATION_PATH, 'Workflow'),
os.path.join(DESTINATION_PATH, 'Workflow/FolderA'),
os.path.join(DESTINATION_PATH, 'Workflow/FolderB')
]
getDir = [os.path.join(DESTINATION_PATH, 'Workflow/FolderA'),
os.path.join(DESTINATION_PATH, 'Workflow/FolderB')
]
removeFile = [os.path.join(DESTINATION_PATH, 'Workflow/FolderA/File1')]
rmdir = [os.path.join(DESTINATION_PATH, 'Workflow')]
# This list is used to check for existance of files
# after uploading the directory: they should NOT exist.
# Uploading a directory does not work.
filesInFolderAandB = []
for dirName in ('Workflow/FolderA', 'Workflow/FolderB'):
for root, _dirs, files in os.walk(os.path.join(local_path, dirName)):
for fn in files:
filesInFolderAandB.append(
os.path.join(
DESTINATION_PATH,
root.replace(
local_path,
'').strip('/'),
fn))
filesInFolderAandB = dict.fromkeys(filesInFolderAandB, False)
fileAdlers = {}
fileSizes = {}
for lfn, localFn in putFile.iteritems():
fileAdlers[lfn] = fileAdler(localFn)
fileSizes[lfn] = getSize(localFn)
clearDirectory(se, local_path, DESTINATION_PATH)
def teardown():
print("Cleaning local test")
shutil.rmtree(local_path)
clearDirectory(se, local_path, DESTINATION_PATH)
request.addfinalizer(teardown)
return local_path, random.randint(0, 100) # provide the fixture value
class StorageElementHandler( RequestHandler ):
"""
.. class:: StorageElementHandler
"""
def __confirmToken( self, token, path, mode ):
""" Confirm the access rights for the path in a given mode
"""
# Not yet implemented
return True
@staticmethod
def __checkForDiskSpace( dpath, size ):
""" Check if the directory dpath can accommodate 'size' volume of data
"""
stats = os.statvfs( dpath )
dsize = stats.f_bsize * stats.f_bavail
maxStorageSizeBytes = MAX_STORAGE_SIZE * 1024 * 1024
return min( dsize, maxStorageSizeBytes ) > size
def __resolveFileID( self, fileID ):
""" get path to file for a given :fileID: """
port = self.getCSOption( 'Port', '' )
if not port:
return ''
if ":%s" % port in fileID:
loc = fileID.find( ":%s" % port )
if loc >= 0:
fileID = fileID[loc + len( ":%s" % port ):]
serviceName = self.serviceInfoDict['serviceName']
loc = fileID.find( serviceName )
if loc >= 0:
fileID = fileID[loc + len( serviceName ):]
loc = fileID.find( '?=' )
if loc >= 0:
fileID = fileID[loc + 2:]
if fileID.find( BASE_PATH ) == 0:
return fileID
while fileID and fileID[0] == '/':
fileID = fileID[1:]
return os.path.join( BASE_PATH, fileID )
@staticmethod
def __getFileStat( path ):
""" Get the file stat information
"""
resultDict = {}
try:
statTuple = os.stat( path )
except OSError, x:
if str( x ).find( 'No such file' ) >= 0:
resultDict['Exists'] = False
return S_OK( resultDict )
else:
return S_ERROR( 'Failed to get metadata for %s' % path )
resultDict['Exists'] = True
mode = statTuple[ST_MODE]
resultDict['Type'] = "File"
resultDict['File'] = True
resultDict['Directory'] = False
if S_ISDIR( mode ):
resultDict['Type'] = "Directory"
resultDict['File'] = False
resultDict['Directory'] = True
resultDict['Size'] = statTuple[ST_SIZE]
resultDict['TimeStamps'] = ( statTuple[ST_ATIME], statTuple[ST_MTIME], statTuple[ST_CTIME] )
resultDict['Cached'] = 1
resultDict['Migrated'] = 0
resultDict['Lost'] = 0
resultDict['Unavailable'] = 0
resultDict['Mode'] = S_IMODE( mode )
if resultDict['File']:
cks = fileAdler( path )
resultDict['Checksum'] = cks
resultDict = StorageBase._addCommonMetadata( resultDict )
return S_OK( resultDict )
def main():
dm = DataManager()
fileTupleBuffer = []
res = getProxyInfo( False, False )
if not res['OK']:
gLogger.error( "Failed to get client proxy information.", res['Message'] )
DIRAC.exit( 2 )
proxyInfo = res['Value']
if proxyInfo['secondsLeft'] == 0:
gLogger.error( "Proxy expired" )
DIRAC.exit( 2 )
username = proxyInfo['username']
vo = ''
if 'group' in proxyInfo:
vo = getVOForGroup( proxyInfo['group'] )
counter = 0
for f in files:
counter += 1
local_f=f
if not f.startswith('/cefs'):
# gLogger.error('File must be under "/cefs"')
# continue
#if the file to reg is not under /cefs, use put and register
folder_name=os.path.basename(os.path.dirname(f))
# lfn = '/cepc/user/%s/%s/jsub/'%(username[0],username) + folder_name + '/' + os.path.basename(f)
lfn = '/cepc/user/%s/%s/jsub/'%(username[0],username) + folder_name + '/' + os.path.basename(f)
# dirname = os.path.dirname(local_f)
# os.system('mkdir -p %s'%(dirname))
# os.system('cp %s %s' %(f,local_f))
do_put_and_register=True
else:
lfn = '/cepc/lustre-ro' + os.path.abspath(f)
do_put_and_register=False
result = fcc.isFile(lfn)
if result['OK'] and lfn in result['Value']['Successful'] and result['Value']['Successful'][lfn]:
continue
size = os.path.getsize(f)
adler32 = fileAdler(f)
guid = makeGuid()
fileTuple = (lfn, local_f, size, _se, guid, adler32)
fileTupleBuffer.append(fileTuple)
gLogger.debug('Register to lfn: %s' % lfn)
gLogger.debug('fileTuple: %s' % (fileTuple,))
if len(fileTupleBuffer) >= _bufferSize:
if do_put_and_register:
result = dm.putAndRegister(lfn, local_f, _se, guid, overwrite=overwrite)
else:
result = dm.registerFile(fileTupleBuffer)
print('register result', result)
# if not result['OK']:
# gLogger.error('Register file failed')
# return 1
del fileTupleBuffer[:]
gLogger.debug('%s files registered' % counter)
if fileTupleBuffer:
if do_put_and_register:
result = dm.putAndRegister(lfn, local_f, _se, guid, overwrite=overwrite)
else:
result = dm.registerFile(fileTupleBuffer)
print('register result', result)
# if not result['OK']:
# gLogger.error('Register file failed')
# return 1
del fileTupleBuffer[:]
gLogger.info('Totally %s files registered' % counter)
return 0
def setuptest(request):
global local_path, download_dir, putDir, createDir, putFile, isFile, listDir, getDir, getFile, rmDir, removeFile, se, filesInFolderAandB, fileAdlers, fileSizes
local_path = tempfile.mkdtemp()
download_dir = os.path.join(local_path, "getFile")
os.mkdir(download_dir)
# create the local structure
workPath = os.path.join(local_path, "Workflow")
os.mkdir(workPath)
os.mkdir(os.path.join(workPath, "FolderA"))
with open(os.path.join(workPath, "FolderA", "FileA"), "w") as f:
f.write(_mul("FileA"))
os.mkdir(os.path.join(workPath, "FolderA", "FolderAA"))
with open(os.path.join(workPath, "FolderA", "FolderAA", "FileAA"),
"w") as f:
f.write(_mul("FileAA"))
os.mkdir(os.path.join(workPath, "FolderB"))
with open(os.path.join(workPath, "FolderB", "FileB"), "w") as f:
f.write(_mul("FileB"))
for fn in ["File1", "File2", "File3"]:
with open(os.path.join(workPath, fn), "w") as f:
f.write(_mul(fn))
# request.param is the SE name one after the other
se = StorageElement(request.param)
putDir = {
os.path.join(DESTINATION_PATH, "Workflow/FolderA"):
os.path.join(local_path, "Workflow/FolderA"),
os.path.join(DESTINATION_PATH, "Workflow/FolderB"):
os.path.join(local_path, "Workflow/FolderB"),
}
createDir = [
os.path.join(DESTINATION_PATH, "Workflow/FolderA/FolderAA"),
os.path.join(DESTINATION_PATH, "Workflow/FolderA/FolderABA"),
os.path.join(DESTINATION_PATH, "Workflow/FolderA/FolderAAB"),
]
putFile = {
os.path.join(DESTINATION_PATH, "Workflow/FolderA/File1"):
os.path.join(local_path, "Workflow/File1"),
os.path.join(DESTINATION_PATH, "Workflow/FolderAA/File1"):
os.path.join(local_path, "Workflow/File1"),
os.path.join(DESTINATION_PATH, "Workflow/FolderBB/File2"):
os.path.join(local_path, "Workflow/File2"),
os.path.join(DESTINATION_PATH, "Workflow/FolderB/File2"):
os.path.join(local_path, "Workflow/File2"),
os.path.join(DESTINATION_PATH, "Workflow/File3"):
os.path.join(local_path, "Workflow/File3"),
}
isFile = list(putFile)
listDir = [
os.path.join(DESTINATION_PATH, "Workflow"),
os.path.join(DESTINATION_PATH, "Workflow/FolderA"),
os.path.join(DESTINATION_PATH, "Workflow/FolderB"),
]
getDir = [
os.path.join(DESTINATION_PATH, "Workflow/FolderA"),
os.path.join(DESTINATION_PATH, "Workflow/FolderB")
]
removeFile = [os.path.join(DESTINATION_PATH, "Workflow/FolderA/File1")]
rmdir = [os.path.join(DESTINATION_PATH, "Workflow")]
# This list is used to check for existance of files
# after uploading the directory: they should NOT exist.
# Uploading a directory does not work.
filesInFolderAandB = []
for dirName in ("Workflow/FolderA", "Workflow/FolderB"):
for root, _dirs, files in os.walk(os.path.join(local_path, dirName)):
for fn in files:
filesInFolderAandB.append(
os.path.join(DESTINATION_PATH,
root.replace(local_path, "").strip("/"), fn))
filesInFolderAandB = dict.fromkeys(filesInFolderAandB, False)
fileAdlers = {}
fileSizes = {}
for lfn, localFn in putFile.items():
fileAdlers[lfn] = fileAdler(localFn)
fileSizes[lfn] = getSize(localFn)
clearDirectory(se, local_path, DESTINATION_PATH)
def teardown():
print("Cleaning local test")
shutil.rmtree(local_path)
clearDirectory(se, local_path, DESTINATION_PATH)
request.addfinalizer(teardown)
return local_path, random.randint(0, 100) # provide the fixture value
