for i in range(len(subdirs)):
subdir = dir + subdirs[i].strip("\n") + "/"
if (options.clean): cleanfiles(subdir)
if (options.duplicate): removeduplicated(subdir)
makefilelists(subdir)
# Transfer Files
myOptions = drOptions()
myOptions.TO_SITE = options.TO_SITE
myOptions.FROM_SITE = "CERN_CASTOR_USER"
myOptions.WHITELIST = ""
myOptions.BLACKLIST = ""
filelists = popen("ls | grep '\.filelist'").readlines()
for i in range(len(filelists)):
filelist = filelists[i].strip("\n")
myOptions.logfile = filelist[0:filelist.find(".")] + ".log"
fileName = [
filelist, options.USERDIR + "/" + filelist[0:filelist.find(".")]
]
#print "fileName is: %s" %fileName
NumFiles = popen("cat " + filelist +
" | wc | awk '{print $1}'").readlines()
NumFiles[0] = NumFiles[0].strip("\n")
myOptions.THREADS = NumFiles[0]
if int(NumFiles[0]) > 25: myOptions.THREADS = "25"
if options.singlethread: myOptions.THREADS = "1"
drExit = data_replica.data_replica(fileName, myOptions)
if (drExit != 0 and not (options.DRYRUN)):
print "Some errors in copying"
def dbs_transferRegister(DATASET, TO_SITE):
#SiteToSe, SeToSite = getSeName()
myBlocks = getDatasetBlockList(api, DATASET)
if myBlocks!=1:
totalDatasetSize = 0
for block in myBlocks:
totalDatasetSize += block['BlockSize']
print "Dataset Size: "+ str(totalDatasetSize/(1024*1024*1024)) +" GB"
blockTransferErrors=0
for block in myBlocks:
skip = False
print "\n------- Copying block: "+ block['Name']
print "Block Size: "+str( float(block['BlockSize'])/(1024*1024*1024))+" GB"
# Checking existing replicas
if not options.RETRANSFER:
for se in block['StorageElementList']:
if SeToSite[ se["Name"] ] == TO_SITE:
print "[INFO] Block already at destination, skipping"
skip = True
if skip:
continue
logfile = "data_replica_"+str(os.getpid())+".log"
fileList = getBlockListFile(api,DATASET,block['Name'])
fileName = createFileTxt( fileList)
myOptions = drOptions()
myOptions.TO_SITE = TO_SITE
myOptions.logfile = logfile
myOptions.DEBUG = options.DEBUG
myOptions.TOOL = options.TOOL
myOptions.delete = options.DELETE
sourceSEs = block['StorageElementList']
data_replica.setBlackWhiteSiteList(options,PREFERRED_SITES, DENIED_SITES )
filteredSourceSEs = arrange_sources(sourceSEs ,PREFERRED_SITES, DENIED_SITES )
if filteredSourceSEs == []:
print "[ERROR] No sites found for block:\n ",block['Name'],"\nplease review your blacklist. Exiting..."
exit(1)
blockTransferred=False
for se in filteredSourceSEs:
print "SE: " +se['Name']
myOptions.FROM_SITE = SeToSite[ se['Name'] ]
print "Copying from "+myOptions.FROM_SITE
drExit = data_replica.data_replica([fileName], myOptions)
if drExit!=0:
print "Some errors in copying, block not added to replica"
### print error files?
else:
print "\nCopy succeded for block "+ block["Name"]+", registering in DBS..."
addBlockReplica(api, block['Name'], SiteToSe[TO_SITE])
blockTransferred=True
print os.getcwd()+"/"+fileName
os.unlink( os.getcwd()+"/"+fileName)
break
if not blockTransferred: blockTransferErrors += 1
print 'Processed %d blocks, %d Copied successfully' % (len(myBlocks), len(myBlocks)-blockTransferErrors)
exitCode = 0 if blockTransferErrors == 0 else 2
return exitCode
if options.recursive:
subdirs = popen("rfdir "+dir+" | awk '{print $9}'").readlines()
for i in range(len(subdirs)):
subdir = dir+subdirs[i].strip("\n") + "/"
if (options.clean): cleanfiles(subdir)
if (options.duplicate): removeduplicated(subdir)
makefilelists(subdir)
# Transfer Files
myOptions = drOptions()
myOptions.TO_SITE = options.TO_SITE
myOptions.FROM_SITE = "CERN_CASTOR_USER"
myOptions.WHITELIST = ""
myOptions.BLACKLIST = ""
filelists = popen("ls | grep '\.filelist'").readlines()
for i in range(len(filelists)):
filelist = filelists[i].strip("\n")
myOptions.logfile = filelist[0:filelist.find(".")]+".log"
fileName = [filelist,options.USERDIR+"/"+filelist[0:filelist.find(".")]]
#print "fileName is: %s" %fileName
NumFiles = popen("cat "+filelist+" | wc | awk '{print $1}'").readlines()
NumFiles[0] = NumFiles[0].strip("\n")
myOptions.THREADS = NumFiles[0]
if int(NumFiles[0])>25: myOptions.THREADS="25"
if options.singlethread: myOptions.THREADS="1"
drExit = data_replica.data_replica(fileName, myOptions)
if (drExit!=0 and not(options.DRYRUN)):
print "Some errors in copying"
