def cp(self, pfn, command):
abspath = os.path.abspath(self.filename)
if cmd_exist("gfal-copy") and self.command in [None, "GFAL"]:
abspath = "file://" + abspath
undoScram = "which scram >/dev/null 2>&1 && eval `scram unsetenv -sh`"
cpcmd = undoScram + "; " + command + abspath + " '" + pfn + "'"
self.logger.info('Executing command: %s' % cpcmd)
self.logger.info('Please wait...')
cpprocess = subprocess.Popen(cpcmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
cpout, cperr = cpprocess.communicate()
cpexitcode = cpprocess.returncode
if cpexitcode:
self.logger.info('Failed running copy command')
if cpout:
self.logger.info(' Stdout:\n %s' %
str(cpout).replace('\n', '\n '))
if cperr:
self.logger.info(' Stderr:\n %s' %
str(cperr).replace('\n', '\n '))
return cpout, cperr, cpexitcode
def cp(self, pfn, command):
abspath = os.path.abspath(self.filename)
if cmd_exist("gfal-copy") and self.command in [None, "GFAL"]:
abspath = "file://" + abspath
cpcmd = command + abspath + " '" + pfn + "'"
self.logger.info('Executing command: %s' % cpcmd)
self.logger.info('Please wait...')
cpprocess = subprocess.Popen(cpcmd, stdout = subprocess.PIPE, stderr = subprocess.PIPE, shell = True)
cpout, cperr = cpprocess.communicate()
cpexitcode = cpprocess.returncode
if cpexitcode:
self.logger.info('Failed running copy command')
if cpout:
self.logger.info(' Stdout:\n %s' % str(cpout).replace('\n', '\n '))
if cperr:
self.logger.info(' Stderr:\n %s' % str(cperr).replace('\n', '\n '))
return cpout, cperr, cpexitcode
def cp(self, pfn, command):
abspath = path.abspath(self.filename)
if cmd_exist("gfal-copy"):
abspath = "file://" + abspath
cpcmd = command + abspath + " '" + pfn + "'"
self.logger.info('Executing command: %s' % cpcmd)
self.logger.info('Please wait...')
cpprocess = subprocess.Popen(cpcmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
cpout, cperr = cpprocess.communicate()
cpexitcode = cpprocess.returncode
if cpexitcode:
self.logger.info('Failed running copy command')
if cpout:
self.logger.info(' Stdout:\n %s' %
str(cpout).replace('\n', '\n '))
if cperr:
self.logger.info(' Stderr:\n %s' %
str(cperr).replace('\n', '\n '))
return cpout, cperr, cpexitcode
def __call__(self):
"""
Copying locally files staged remotely.
*Using a subprocess to encapsulate the copy command.
* maximum parallel download is 10, line 61
* default --sendreceive-timeout is 1800 s, line 75 and 77
"""
globalExitcode = -1
dicttocopy = self.options.inputdict
# taking number of parallel download to create from user, default is 10
if self.options.nparallel == None:
nsubprocess = 10
else:
nsubprocess = int(self.options.nparallel)
if nsubprocess <= 0 or nsubprocess > 20:
self.logger.info("Inappropriate number of parallel download, must between 0 to 20 ")
return -1
command = ""
if cmd_exist("gfal-copy"):
self.logger.info("Will use `gfal-copy` command for file transfers")
command = "env -i gfal-copy -v "
command += " -T "
elif cmd_exist("lcg-cp"):
self.logger.info("Will use `lcg-cp` command for file transfers")
command = "lcg-cp --connect-timeout 20 --verbose -b -D srmv2"
command += " --sendreceive-timeout "
else:
# This should not happen. If it happens, Site Admin have to install GFAL2 (yum install gfal2-util gfal2-all)
self.logger.info("%sError%s: Can`t find command `gfal-copy` or `lcg-ls`, Please contact the site administrator." % (colors.RED, colors.NORMAL))
return [], []
command += "1800" if self.options.waittime == None else str(1800 + int(self.options.waittime))
# timeout = 20 + 240 + 60 #giving 1 extra minute: 5min20"
srmtimeout = 900 # default transfer timeout in case the file size is unknown: 15min
minsrmtimeout = 60 # timeout cannot be less then 1min
downspeed = float(250*1024) # default speed assumes a download of 250KB/s
mindownspeed = 20*1024.
manager = Manager()
successfiles = manager.dict()
failedfiles = manager.dict()
self.logger.debug("Starting ChildProcess with %s ChildProcess" % nsubprocess)
inputq, processarray = self.startchildproc(self.processWorker,nsubprocess, successfiles, failedfiles)
for myfile in dicttocopy:
if downspeed < mindownspeed:
downspeed = mindownspeed
fileid = myfile['pfn'].split('/')[-1]
dirpath = os.path.join(self.options.destination, myfile['suffix'] if 'suffix' in myfile else '')
url_input = bool(re.match("^[a-z]+://", dirpath))
if not url_input and not os.path.isdir(dirpath):
os.makedirs(dirpath)
localFilename = os.path.join(dirpath, str(fileid))
##### Handling the "already existing file" use case
if not url_input and os.path.isfile(localFilename):
size = os.path.getsize(localFilename)
# delete the file if its size is zero or its size is not the expected size
if size == 0 or ('size' in myfile and myfile['size'] != size):
try:
self.logger.info("Removing %s as it is not complete: current size %s, expected size %s" % (fileid, size, \
myfile['size'] if 'size' in myfile else 'unknown'))
os.remove(localFilename)
except Exception, ex:
self.logger.info("%sError%s: Cannot remove the file because of: %s" % (colors.RED, colors.NORMAL,ex))
# if the file still exists skip it
if not url_input and os.path.isfile(localFilename):
self.logger.info("Skipping %s as file already exists in %s" % (fileid, localFilename))
continue
##### Creating the command
# timeout based on file size and download speed * 2
maxtime = srmtimeout if not 'size' in myfile or myfile['size'] == 0 else int(ceil(2*myfile['size']/downspeed))
localsrmtimeout = minsrmtimeout if maxtime < minsrmtimeout else maxtime # do not want a too short timeout
timeout = " --srm-timeout "
if cmd_exist("gfal-copy"):
timeout = " -t "
cmd = '%s %s %s %%s' % (command, timeout + str(localsrmtimeout) + ' ', myfile['pfn'])
if url_input:
cmd = cmd % localFilename
else:
cmd = cmd % ("file://%s" % localFilename)
self.logger.info("Placing file '%s' in retrieval queue " % fileid)
inputq.put((myfile, cmd))
def __call__(self):
"""
Copying locally files staged remotely.
* using a subprocess to encapsulate the copy command.
* maximum parallel download is 10
"""
## This is the log gilename that is going to be used by the subprocesses that copy the file
## Using the same logfile is not supported automatically, see:
## https://docs.python.org/2/howto/logging-cookbook.html#logging-to-a-single-file-from-multiple-processes
self.remotecpLogile = "%s/remote_copy.log" % os.path.dirname(
self.logger.logfile)
dicttocopy = self.options.inputdict
# taking number of parallel download to create from user, default is 10
if self.options.nparallel == None:
nsubprocess = 10
else:
nsubprocess = int(self.options.nparallel)
if nsubprocess <= 0 or nsubprocess > 20:
self.logger.info(
"Inappropriate number of parallel download, must between 0 to 20 "
)
return -1
command = ""
if cmd_exist("gfal-copy") and self.options.command not in ["LCG"]:
self.logger.info("Will use `gfal-copy` command for file transfers")
command = "gfal-copy -v "
if self.options.checksum:
command += "-K %s " % self.options.checksum
command += " -T "
elif cmd_exist("lcg-cp") and self.options.command not in ["GFAL"]:
self.logger.info("Will use `lcg-cp` command for file transfers")
command = "lcg-cp --connect-timeout 20 --verbose -b -D srmv2"
if self.options.checksum:
command += " --checksum-type %s " % self.options.checksum
command += " --sendreceive-timeout "
else:
# This should not happen. If it happens, Site Admin have to install GFAL2 (yum install gfal2-util gfal2-all)
self.logger.info(
"%sError%s: Can`t find command `gfal-copy` or `lcg-ls`, Please contact the site administrator."
% (colors.RED, colors.NORMAL))
return [], []
command += "1800" if self.options.waittime == None else str(
1800 + int(self.options.waittime))
# timeout = 20 + 240 + 60 #giving 1 extra minute: 5min20"
srmtimeout = 900 # default transfer timeout in case the file size is unknown: 15min
minsrmtimeout = 60 # timeout cannot be less then 1min
downspeed = float(250 *
1024) # default speed assumes a download of 250KB/s
mindownspeed = 20 * 1024.
manager = Manager()
successfiles = manager.dict()
failedfiles = manager.dict()
self.logger.debug("Starting ChildProcess with %s ChildProcess" %
nsubprocess)
inputq, processarray = self.startchildproc(self.processWorker,
nsubprocess, successfiles,
failedfiles)
for myfile in dicttocopy:
if downspeed < mindownspeed:
downspeed = mindownspeed
fileid = myfile['pfn'].split('/')[-1]
dirpath = os.path.join(
self.options.destination,
myfile['suffix'] if 'suffix' in myfile else '')
url_input = bool(re.match("^[a-z]+://", dirpath))
if not url_input and not os.path.isdir(dirpath):
os.makedirs(dirpath)
localFilename = os.path.join(dirpath, str(fileid))
##### Handling the "already existing file" use case
if not url_input and os.path.isfile(localFilename):
size = os.path.getsize(localFilename)
# delete the file if its size is zero or its size is not the expected size
if size == 0 or ('size' in myfile and myfile['size'] != size):
try:
self.logger.info("Removing %s as it is not complete: current size %s, expected size %s" % (fileid, size, \
myfile['size'] if 'size' in myfile else 'unknown'))
os.remove(localFilename)
except OSError as ex:
self.logger.info(
"%sError%s: Cannot remove the file because of: %s"
% (colors.RED, colors.NORMAL, ex))
# if the file still exists skip it
if not url_input and os.path.isfile(localFilename):
self.logger.info("Skipping %s as file already exists in %s" %
(fileid, localFilename))
continue
##### Creating the command
# better to execut grid commands in the pre-CMS environment
undoScram = "which scram >/dev/null 2>&1 && eval `scram unsetenv -sh`"
# timeout based on file size and download speed * 2
maxtime = srmtimeout if not 'size' in myfile or myfile[
'size'] == 0 else int(ceil(2 * myfile['size'] / downspeed))
localsrmtimeout = minsrmtimeout if maxtime < minsrmtimeout else maxtime # do not want a too short timeout
timeout = " --srm-timeout "
if cmd_exist("gfal-copy") and self.options.command not in ["LCG"]:
timeout = " -t "
cmd = undoScram + '; %s %s %s %%s' % (
command, timeout + str(localsrmtimeout) + ' ', myfile['pfn'])
if url_input:
cmd = cmd % localFilename
else:
cmd = cmd % ("file://%s" % localFilename)
self.logger.info("Placing file '%s' in retrieval queue " % fileid)
inputq.put((myfile, cmd))
self.logger.info("Please wait")
keybInt = self.stopchildproc(inputq, processarray, nsubprocess)
self.saveSubprocessesOut(failedfiles, keybInt)
if keybInt:
## if ctrl-C was hit we wont find anything interesting in the subprocesses out
## that means that successfiles and failedfiles will not be dict as normally expected
return [], []
elif len(successfiles) == 0:
self.logger.info("No file retrieved")
elif len(failedfiles) != 0:
self.logger.info(colors.GREEN +
"Number of files successfully retrieved: %s" %
len(successfiles) + colors.NORMAL)
self.logger.info(colors.RED +
"Number of files failed to be retrieved: %s" %
len(failedfiles) + colors.NORMAL)
#self.logger.debug("List of failed file and reason: %s" % failedfiles)
else:
self.logger.info("%sSuccess%s: All files successfully retrieved" %
(colors.GREEN, colors.NORMAL))
return successfiles, failedfiles
def __call__(self):
"""
Copying locally files staged remotely.
* using a subprocess to encapsulate the copy command.
* maximum parallel download is 10
"""
## This is the log gilename that is going to be used by the subprocesses that copy the file
## Using the same logfile is not supported automatically, see:
## https://docs.python.org/2/howto/logging-cookbook.html#logging-to-a-single-file-from-multiple-processes
self.remotecpLogile = "%s/remote_copy.log" % os.path.dirname(self.logger.logfile)
dicttocopy = self.options.inputdict
# taking number of parallel download to create from user, default is 10
if self.options.nparallel == None:
nsubprocess = 10
else:
nsubprocess = int(self.options.nparallel)
if nsubprocess <= 0 or nsubprocess > 20:
self.logger.info("Inappropriate number of parallel download, must between 0 to 20 ")
return -1
command = ""
if cmd_exist("gfal-copy") and self.options.command not in ["LCG"]:
self.logger.info("Will use `gfal-copy` command for file transfers")
command = "env -i X509_USER_PROXY=%s gfal-copy -v " % os.path.abspath(self.proxyfilename)
if self.options.checksum:
command += "-K %s " % self.options.checksum
command += " -T "
elif cmd_exist("lcg-cp") and self.options.command not in ["GFAL"]:
self.logger.info("Will use `lcg-cp` command for file transfers")
command = "lcg-cp --connect-timeout 20 --verbose -b -D srmv2"
if self.options.checksum:
command += " --checksum-type %s " % self.options.checksum
command += " --sendreceive-timeout "
else:
# This should not happen. If it happens, Site Admin have to install GFAL2 (yum install gfal2-util gfal2-all)
self.logger.info("%sError%s: Can`t find command `gfal-copy` or `lcg-ls`, Please contact the site administrator." % (colors.RED, colors.NORMAL))
return [], []
command += "1800" if self.options.waittime == None else str(1800 + int(self.options.waittime))
# timeout = 20 + 240 + 60 #giving 1 extra minute: 5min20"
srmtimeout = 900 # default transfer timeout in case the file size is unknown: 15min
minsrmtimeout = 60 # timeout cannot be less then 1min
downspeed = float(250*1024) # default speed assumes a download of 250KB/s
mindownspeed = 20*1024.
manager = Manager()
successfiles = manager.dict()
failedfiles = manager.dict()
self.logger.debug("Starting ChildProcess with %s ChildProcess" % nsubprocess)
inputq, processarray = self.startchildproc(self.processWorker, nsubprocess, successfiles, failedfiles)
for myfile in dicttocopy:
if downspeed < mindownspeed:
downspeed = mindownspeed
fileid = myfile['pfn'].split('/')[-1]
dirpath = os.path.join(self.options.destination, myfile['suffix'] if 'suffix' in myfile else '')
url_input = bool(re.match("^[a-z]+://", dirpath))
if not url_input and not os.path.isdir(dirpath):
os.makedirs(dirpath)
localFilename = os.path.join(dirpath, str(fileid))
##### Handling the "already existing file" use case
if not url_input and os.path.isfile(localFilename):
size = os.path.getsize(localFilename)
# delete the file if its size is zero or its size is not the expected size
if size == 0 or ('size' in myfile and myfile['size'] != size):
try:
self.logger.info("Removing %s as it is not complete: current size %s, expected size %s" % (fileid, size, \
myfile['size'] if 'size' in myfile else 'unknown'))
os.remove(localFilename)
except OSError as ex:
self.logger.info("%sError%s: Cannot remove the file because of: %s" % (colors.RED, colors.NORMAL, ex))
# if the file still exists skip it
if not url_input and os.path.isfile(localFilename):
self.logger.info("Skipping %s as file already exists in %s" % (fileid, localFilename))
continue
##### Creating the command
# timeout based on file size and download speed * 2
maxtime = srmtimeout if not 'size' in myfile or myfile['size'] == 0 else int(ceil(2*myfile['size']/downspeed))
localsrmtimeout = minsrmtimeout if maxtime < minsrmtimeout else maxtime # do not want a too short timeout
timeout = " --srm-timeout "
if cmd_exist("gfal-copy") and self.options.command not in ["LCG"]:
timeout = " -t "
cmd = '%s %s %s %%s' % (command, timeout + str(localsrmtimeout) + ' ', myfile['pfn'])
if url_input:
cmd = cmd % localFilename
else:
cmd = cmd % ("file://%s" % localFilename)
self.logger.info("Placing file '%s' in retrieval queue " % fileid)
inputq.put((myfile, cmd))
self.logger.info("Please wait")
keybInt = self.stopchildproc(inputq, processarray, nsubprocess)
self.saveSubprocessesOut(failedfiles, keybInt)
if keybInt:
## if ctrl-C was hit we wont find anything interesting in the subprocesses out
## that means that successfiles and failedfiles will not be dict as normally expected
return [], []
elif len(successfiles) == 0:
self.logger.info("No file retrieved")
elif len(failedfiles) != 0:
self.logger.info(colors.GREEN+"Number of files successfully retrieved: %s" % len(successfiles)+colors.NORMAL)
self.logger.info(colors.RED+"Number of files failed to be retrieved: %s" % len(failedfiles)+colors.NORMAL)
#self.logger.debug("List of failed file and reason: %s" % failedfiles)
else:
self.logger.info("%sSuccess%s: All files successfully retrieved" % (colors.GREEN,colors.NORMAL))
return successfiles , failedfiles
def __call__(self):
username = None
if hasattr(self.options, 'userlfn') and self.options.userlfn != None:
self.lfnsaddprefix = self.options.userlfn
else:
## If the user didn't provide an LFN path where to check the write permission,
## assume he/she wants to check in /store/user/<username>. Retrieve his/her
## username from SiteDB.
self.logger.info('Will check write permission in the default location /store/user/<username>')
username = getUserDNandUsernameFromSiteDB(self.logger).get('username')
if username:
self.lfnsaddprefix = '/store/user/' + username
else:
return {'status': 'FAILED'}
## Check that the location where we want to check write permission
## is one where the user will be allowed to stageout.
self.logger.info("Validating LFN %s..." % (self.lfnsaddprefix))
msg = "Refusing to check write permission in %s, because this is not an allowed LFN for stageout." % (self.lfnsaddprefix)
msg += "\nThe LFN must start with either"
msg += " '/store/user/<username>/' or '/store/group/<groupname>/'"
msg += " (or '/store/local/<something>/' if publication is off),"
msg += " where username is your username as registered in SiteDB"
msg += " (i.e. the username of your CERN primary account)."
msg += "\nLFN %s is not valid." % (self.lfnsaddprefix)
if not username and self.lfnsaddprefix.startswith('/store/user/'):
username = getUserDNandUsernameFromSiteDB(self.logger).get('username')
if not checkOutLFN(self.lfnsaddprefix, username):
self.logger.info(msg)
return {'status': 'FAILED'}
else:
self.logger.info("LFN %s is valid." % (self.lfnsaddprefix))
cp_cmd = ""
if cmd_exist("gfal-copy") and cmd_exist("gfal-rm") and self.command in [None, "GFAL"]:
self.logger.info("Will use `gfal-copy`, `gfal-rm` commands for checking write permissions")
cp_cmd = "env -i X509_USER_PROXY=%s gfal-copy -p -v -t 180 " % os.path.abspath(self.proxyfilename)
delfile_cmd = "env -i X509_USER_PROXY=%s gfal-rm -v -t 180 " % os.path.abspath(self.proxyfilename)
deldir_cmd = "env -i X509_USER_PROXY=%s gfal-rm -r -v -t 180 " % os.path.abspath(self.proxyfilename)
if self.checksum:
cp_cmd += "-K %s " % self.checksum
elif cmd_exist("lcg-cp") and cmd_exist("lcg-del"):
self.logger.info("Will use `lcg-cp`, `lcg-del` commands for checking write permissions")
cp_cmd = "lcg-cp -v -b -D srmv2 --connect-timeout 180 "
delfile_cmd = "lcg-del --connect-timeout 180 -b -l -D srmv2 "
deldir_cmd = "lcg-del -d --connect-timeout 180 -b -l -D srmv2 "
if self.checksum:
cp_cmd += "--checksum-type %s " % self.checksum
else:
self.logger.info("Neither gfal nor lcg command was found")
return {'status': 'FAILED'}
self.logger.info('Will check write permission in %s on site %s' % (self.lfnsaddprefix, self.options.sitename))
timestamp = str(time.strftime("%Y%m%d_%H%M%S"))
self.filename = 'crab3checkwrite_' + timestamp + '.tmp'
self.subdir = 'crab3checkwrite_' + timestamp
self.createFile()
pfn = self.getPFN()
dirpfn = pfn[:len(pfn)-len(self.filename)]
self.logger.info('\nAttempting to create (dummy) directory %s and copy (dummy) file %s to %s\n' % (self.subdir, self.filename, self.lfnsaddprefix))
cpout, cperr, cpexitcode = self.cp(pfn, cp_cmd)
if cpexitcode == 0:
self.logger.info('\nSuccessfully created directory %s and copied file %s to %s' % (self.subdir, self.filename, self.lfnsaddprefix))
self.logger.info('\nAttempting to delete file %s\n' % (pfn))
delexitcode = self.delete(pfn, delfile_cmd)
if delexitcode:
self.logger.info('\nFailed to delete file %s' % (pfn))
finalmsg = '%sError%s: CRAB3 is able to copy but unable to delete file in %s on site %s. Asynchronous Stage Out with CRAB3 will fail.' % (colors.RED, colors.NORMAL, self.lfnsaddprefix, self.options.sitename)
finalmsg += '\n You may want to contact the site administrators sending them the \'crab checkwrite\' output as printed above.'
returndict = {'status': 'FAILED'}
else:
self.logger.info('\nSuccessfully deleted file %s' % (pfn))
self.logger.info('\nAttempting to delete directory %s\n' % (dirpfn))
delexitcode = self.delete(dirpfn, deldir_cmd)
if delexitcode:
self.logger.info('\nFailed to delete directory %s' % (dirpfn))
finalmsg = '%sError%s: CRAB3 is able to copy but unable to delete directory in %s on site %s. Asynchronous Stage Out with CRAB3 will fail.' % (colors.RED, colors.NORMAL, self.lfnsaddprefix, self.options.sitename)
finalmsg += '\n You may want to contact the site administrators sending them the \'crab checkwrite\' output as printed above.'
returndict = {'status': 'FAILED'}
else:
self.logger.info('\nSuccessfully deleted directory %s' % (dirpfn))
finalmsg = '%sSuccess%s: Able to write in %s on site %s' % (colors.GREEN, colors.NORMAL, self.lfnsaddprefix, self.options.sitename)
returndict = {'status': 'SUCCESS'}
else:
if 'Permission denied' in cperr or 'mkdir: cannot create directory' in cperr:
finalmsg = '%sError%s: Unable to write in %s on site %s' % (colors.RED, colors.NORMAL, self.lfnsaddprefix, self.options.sitename)
finalmsg += '\n You may want to contact the site administrators sending them the \'crab checkwrite\' output as printed above.'
returndict = {'status': 'FAILED'}
elif 'timeout' in cpout or 'timeout' in cperr:
self.logger.info('Connection time out.')
finalmsg = '\nUnable to check write permission in %s on site %s' % (self.lfnsaddprefix, self.options.sitename)
finalmsg += '\nPlease try again later or contact the site administrators sending them the \'crab checkwrite\' output as printed above.'
returndict = {'status': 'FAILED'}
else:
finalmsg = 'Unable to check write permission in %s on site %s' % (self.lfnsaddprefix, self.options.sitename)
finalmsg += '\nPlease try again later or contact the site administrators sending them the \'crab checkwrite\' output as printed above.'
returndict = {'status' : 'FAILED'}
self.removeFile()
self.logger.info('\nCheckwrite Result:')
self.logger.info(finalmsg)
if returndict['status'] == 'FAILED':
self.logger.info('%sNote%s: You cannot write to a site if you did not ask permission.' % (colors.BOLD, colors.NORMAL))
return returndict
def __call__(self):
username = None
if hasattr(self.options, 'userlfn') and self.options.userlfn != None:
self.lfnsaddprefix = self.options.userlfn
else:
## If the user didn't provide an LFN path where to check the write permission,
## assume he/she wants to check in /store/user/<username>. Retrieve his/her
## username from SiteDB.
self.logger.info('Will check write permission in the default location /store/user/<username>')
username = getUserDNandUsernameFromSiteDB(self.logger).get('username')
if username:
self.lfnsaddprefix = '/store/user/' + username
else:
return {'status': 'FAILED'}
## Check that the location where we want to check write permission
## is one where the user will be allowed to stageout.
self.logger.info("Validating LFN %s..." % (self.lfnsaddprefix))
msg = "Refusing to check write permission in %s, because this is not an allowed LFN for stageout." % (self.lfnsaddprefix)
msg += "\nThe LFN must start with either"
msg += " '/store/user/<username>/' or '/store/group/<groupname>/'"
msg += " (or '/store/local/<something>/' if publication is off),"
msg += " where username is your username as registered in SiteDB"
msg += " (i.e. the username of your CERN primary account)."
msg += "\nLFN %s is not valid." % (self.lfnsaddprefix)
if not username and self.lfnsaddprefix.startswith('/store/user/'):
username = getUserDNandUsernameFromSiteDB(self.logger).get('username')
if not checkOutLFN(self.lfnsaddprefix, username):
self.logger.info(msg)
return {'status': 'FAILED'}
else:
self.logger.info("LFN %s is valid." % (self.lfnsaddprefix))
cp_cmd = ""
del_cmd = ""
if cmd_exist("gfal-copy") and cmd_exist("gfal-rm"):
self.logger.info("Will use `gfal-copy`, `gfal-rm` commands for checking write permissions")
cp_cmd = "env -i gfal-copy -v -t 180 "
del_cmd = "env -i gfal-rm -v -t 180 "
elif cmd_exist("lcg-cp") and cmd_exist("lcg-del"):
self.logger.info("Will use `lcg-cp`, `lcg-del` commands for checking write permissions")
cp_cmd = "lcg-cp -v -b -D srmv2 --connect-timeout 180 "
del_cmd = "lcg-del --connect-timeout 180 -b -l -D srmv2 "
else:
self.logger.info("Neither gfal nor lcg command was found")
return {'status': 'FAILED'}
self.logger.info('Will check write permission in %s on site %s' % (self.lfnsaddprefix, self.options.sitename))
retry = 0
stop = False
use_new_file = True
while not stop:
if use_new_file:
self.filename = 'crab3checkwrite.' + str(retry) + '.tmp'
self.createFile()
pfn = self.getPFN()
self.logger.info('Attempting to copy (dummy) file %s to %s on site %s' % (self.filename, self.lfnsaddprefix, self.options.sitename))
cpout, cperr, cpexitcode = self.cp(pfn, cp_cmd)
if cpexitcode == 0:
self.logger.info('Successfully copied file %s to %s on site %s' % (self.filename, self.lfnsaddprefix, self.options.sitename))
self.logger.info('Attempting to delete file %s from site %s' % (pfn, self.options.sitename))
delexitcode = self.delete(pfn, del_cmd)
if delexitcode:
self.logger.info('%sWarning%s: Failed to delete file %s from site %s' % (colors.RED, colors.NORMAL, pfn, self.options.sitename))
else:
self.logger.info('Successfully deleted file %s from site %s' % (pfn, self.options.sitename))
self.logger.info('%sSuccess%s: Able to write in %s on site %s' % (colors.GREEN, colors.NORMAL, self.lfnsaddprefix, self.options.sitename))
returndict = {'status': 'SUCCESS'}
stop = True
else:
if 'Permission denied' in cperr or 'mkdir: cannot create directory' in cperr:
msg = '%sError%s: Unable to write in %s on site %s' % (colors.RED, colors.NORMAL, self.lfnsaddprefix, self.options.sitename)
msg += '\n You may want to contact the site administrators sending them the \'crab checkwrite\' output as printed above.'
self.logger.info(msg)
returndict = {'status': 'FAILED'}
stop = True
elif 'timeout' in cpout or 'timeout' in cperr:
self.logger.info('Connection time out.')
msg = 'Unable to check write permission in %s on site %s' % (self.lfnsaddprefix, self.options.sitename)
msg += '\nPlease try again later or contact the site administrators sending them the \'crab checkwrite\' output as printed above.'
self.logger.info(msg)
returndict = {'status': 'FAILED'}
stop = True
elif 'exist' in cpout or 'exist' in cperr and retry == 0:
self.logger.info('Error copying file %s to %s on site %s; it may be that file already exists.' % (self.filename, self.lfnsaddprefix, self.options.sitename))
self.logger.info('Attempting to delete file %s from site %s' % (pfn, self.options.sitename))
delexitcode = self.delete(pfn, del_cmd)
if delexitcode:
self.logger.info('Failed to delete file %s from site %s' % (pfn, self.options.sitename))
use_new_file = True
else:
self.logger.info('Successfully deleted file %s from site %s' % (pfn, self.options.sitename))
use_new_file = False
retry += 1
else:
msg = 'Unable to check write permission in %s on site %s' % (self.lfnsaddprefix, self.options.sitename)
msg += '\nPlease try again later or contact the site administrators sending them the \'crab checkwrite\' output as printed above.'
self.logger.info(msg)
returndict = {'status' : 'FAILED'}
stop = True
if stop or use_new_file:
self.removeFile()
self.logger.info('%sNote%s: You cannot write to a site if you did not ask permission.' % (colors.BOLD, colors.NORMAL))
return returndict
def __call__(self):
username = None
if hasattr(self.options, 'userlfn') and self.options.userlfn != None:
self.lfnsaddprefix = self.options.userlfn
else:
## If the user didn't provide an LFN path where to check the write permission,
## assume he/she wants to check in /store/user/<username>. Retrieve his/her
## username from SiteDB.
self.logger.info(
'Will check write permission in the default location /store/user/<username>'
)
username = getUserDNandUsernameFromSiteDB(
self.logger).get('username')
if username:
self.lfnsaddprefix = '/store/user/' + username
else:
return {'status': 'FAILED'}
## Check that the location where we want to check write permission
## is one where the user will be allowed to stageout.
self.logger.info("Validating LFN %s..." % (self.lfnsaddprefix))
msg = "Refusing to check write permission in %s, because this is not an allowed LFN for stageout." % (
self.lfnsaddprefix)
msg += "\nThe LFN must start with either '/store/user/<username>/', '/store/group/<groupname>[/<subgroupname>]*/<username>/' or '/store/local/<dirname>',"
msg += " where username is your username as registered in SiteDB (i.e. the username of your CERN primary account)."
msg += "\nLFN %s is not valid." % (self.lfnsaddprefix)
if not username:
username = getUserDNandUsernameFromSiteDB(
self.logger).get('username')
if not checkOutLFN(self.lfnsaddprefix, username):
self.logger.info(msg)
return {'status': 'FAILED'}
else:
self.logger.info("LFN %s is valid." % (self.lfnsaddprefix))
cp_cmd = ""
del_cmd = ""
if cmd_exist("gfal-copy") and cmd_exist("gfal-rm"):
self.logger.info(
"Will use `gfal-copy`, `gfal-rm` commands for checking write permissions"
)
cp_cmd = "env -i gfal-copy -v -t 180 "
del_cmd = "env -i gfal-rm -v -t 180 "
elif cmd_exist("lcg-cp") and cmd_exist("lcg-del"):
self.logger.info(
"Will use `lcg-cp`, `lcg-del` commands for checking write permissions"
)
cp_cmd = "lcg-cp -v -b -D srmv2 --connect-timeout 180 "
del_cmd = "lcg-del --connect-timeout 180 -b -l -D srmv2 "
else:
self.logger.info("Neither gfal nor lcg command was found")
return {'status': 'FAILED'}
self.logger.info('Will check write permission in %s on site %s' %
(self.lfnsaddprefix, self.options.sitename))
retry = 0
stop = False
use_new_file = True
while not stop:
if use_new_file:
self.filename = 'crab3checkwrite.' + str(retry) + '.tmp'
self.createFile()
pfn = self.getPFN()
self.logger.info(
'Attempting to copy (dummy) file %s to %s on site %s' %
(self.filename, self.lfnsaddprefix, self.options.sitename))
cpout, cperr, cpexitcode = self.cp(pfn, cp_cmd)
if cpexitcode == 0:
self.logger.info(
'Successfully copied file %s to %s on site %s' %
(self.filename, self.lfnsaddprefix, self.options.sitename))
self.logger.info('Attempting to delete file %s from site %s' %
(pfn, self.options.sitename))
delexitcode = self.delete(pfn, del_cmd)
if delexitcode:
self.logger.info(
'%sWarning%s: Failed to delete file %s from site %s' %
(colors.RED, colors.NORMAL, pfn,
self.options.sitename))
else:
self.logger.info(
'Successfully deleted file %s from site %s' %
(pfn, self.options.sitename))
self.logger.info(
'%sSuccess%s: Able to write in %s on site %s' %
(colors.GREEN, colors.NORMAL, self.lfnsaddprefix,
self.options.sitename))
returndict = {'status': 'SUCCESS'}
stop = True
else:
if 'Permission denied' in cperr or 'mkdir: cannot create directory' in cperr:
msg = '%sError%s: Unable to write in %s on site %s' % (
colors.RED, colors.NORMAL, self.lfnsaddprefix,
self.options.sitename)
msg += '\n You may want to contact the site administrators sending them the \'crab checkwrite\' output as printed above.'
self.logger.info(msg)
returndict = {'status': 'FAILED'}
stop = True
elif 'timeout' in cpout or 'timeout' in cperr:
self.logger.info('Connection time out.')
msg = 'Unable to check write permission in %s on site %s' % (
self.lfnsaddprefix, self.options.sitename)
msg += '\nPlease try again later or contact the site administrators sending them the \'crab checkwrite\' output as printed above.'
self.logger.info(msg)
returndict = {'status': 'FAILED'}
stop = True
elif 'exist' in cpout or 'exist' in cperr and retry == 0:
self.logger.info(
'Error copying file %s to %s on site %s; it may be that file already exists.'
% (self.filename, self.lfnsaddprefix,
self.options.sitename))
self.logger.info(
'Attempting to delete file %s from site %s' %
(pfn, self.options.sitename))
delexitcode = self.delete(pfn, del_cmd)
if delexitcode:
self.logger.info(
'Failed to delete file %s from site %s' %
(pfn, self.options.sitename))
use_new_file = True
else:
self.logger.info(
'Successfully deleted file %s from site %s' %
(pfn, self.options.sitename))
use_new_file = False
retry += 1
else:
msg = 'Unable to check write permission in %s on site %s' % (
self.lfnsaddprefix, self.options.sitename)
msg += '\nPlease try again later or contact the site administrators sending them the \'crab checkwrite\' output as printed above.'
self.logger.info(msg)
returndict = {'status': 'FAILED'}
stop = True
if stop or use_new_file:
self.removeFile()
self.logger.info(
'%sNote%s: You cannot write to a site if you did not ask permission.'
% (colors.BOLD, colors.NORMAL))
return returndict
def __call__(self):
username = getUsername(self.proxyfilename, logger=self.logger)
if hasattr(self.options, 'userlfn') and self.options.userlfn != None:
self.lfnPrefix = self.options.userlfn
else:
## If the user didn't provide an LFN path where to check the write permission,
## assume he/she wants to check in /store/user/<username>
self.logger.info(
'Will check write permission in the default location /store/user/<username>'
)
self.lfnPrefix = '/store/user/' + username
## Check that the location where we want to check write permission
## is one where the user will be allowed to stageout.
self.logger.info("Validating LFN %s...", self.lfnPrefix)
# if an error message is needed later, prepare it now to keep code below tidy
errMsg = "Refusing to check write permission in %s, because this is not an allowed LFN for stageout." % (
self.lfnPrefix)
errMsg += "\nThe LFN must start with either"
errMsg += " '/store/user/<username>/' or '/store/group/<groupname>/'"
errMsg += " (or '/store/local/<something>/' if publication is off),"
errMsg += " where username is your username as registered in CMS"
errMsg += " (i.e. the username of your CERN primary account)."
errMsg += "\nLFN %s is not valid." % (self.lfnPrefix)
if not checkOutLFN(self.lfnPrefix, username):
self.logger.info(errMsg)
return {'status': 'FAILED'}
else:
self.logger.info("LFN %s is valid.", self.lfnPrefix)
cp_cmd = ""
if cmd_exist("gfal-copy") and cmd_exist(
"gfal-rm") and self.command in [None, "GFAL"]:
self.logger.info(
"Will use `gfal-copy`, `gfal-rm` commands for checking write permissions"
)
cp_cmd = "gfal-copy -p -v -t 180 "
delfile_cmd = "gfal-rm -v -t 180 "
deldir_cmd = "gfal-rm -r -v -t 180 "
if self.checksum:
cp_cmd += "-K %s " % self.checksum
elif cmd_exist("lcg-cp") and cmd_exist("lcg-del"):
self.logger.info(
"Will use `lcg-cp`, `lcg-del` commands for checking write permissions"
)
cp_cmd = "lcg-cp -v -b -D srmv2 --connect-timeout 180 "
delfile_cmd = "lcg-del --connect-timeout 180 -b -l -D srmv2 "
deldir_cmd = "lcg-del -d --connect-timeout 180 -b -l -D srmv2 "
if self.checksum:
cp_cmd += "--checksum-type %s " % self.checksum
else:
self.logger.info("Neither gfal nor lcg command was found")
return {'status': 'FAILED'}
self.logger.info('Will check write permission in %s on site %s',
self.lfnPrefix, self.options.sitename)
timestamp = str(time.strftime("%Y%m%d_%H%M%S"))
self.filename = 'crab3checkwrite_' + timestamp + '.tmp'
self.subdir = 'crab3checkwrite_' + timestamp
lfn = self.lfnPrefix + '/' + self.subdir + '/' + self.filename
site = self.options.sitename
try:
pfn = self.getPFN(site=site, lfn=lfn, username=username)
except Exception:
return {'status': 'FAILED'}
self.createFile()
self.logger.info("Will use PFN: %s", pfn)
dirpfn = pfn[:len(pfn) - len(self.filename)]
self.logger.info(
'\nAttempting to create (dummy) directory %s and copy (dummy) file %s to %s\n'
% (self.subdir, self.filename, self.lfnPrefix))
cpout, cperr, cpexitcode = self.cp(pfn, cp_cmd)
if cpexitcode == 0:
self.logger.info(
'\nSuccessfully created directory %s and copied file %s to %s'
% (self.subdir, self.filename, self.lfnPrefix))
self.logger.info('\nAttempting to delete file %s\n' % (pfn))
delexitcode = self.delete(pfn, delfile_cmd)
if delexitcode:
self.logger.info('\nFailed to delete file %s' % (pfn))
finalmsg = '%sError%s: CRAB3 is able to copy but unable to delete file in %s on site %s. Asynchronous Stage Out with CRAB3 will fail.' % (
colors.RED, colors.NORMAL, self.lfnPrefix,
self.options.sitename)
finalmsg += '\n You may want to contact the site administrators sending them the \'crab checkwrite\' output as printed above.'
returndict = {'status': 'FAILED'}
else:
self.logger.info('\nSuccessfully deleted file %s' % (pfn))
self.logger.info('\nAttempting to delete directory %s\n' %
(dirpfn))
delexitcode = self.delete(dirpfn, deldir_cmd)
if delexitcode:
self.logger.info('\nFailed to delete directory %s' %
(dirpfn))
finalmsg = '%sError%s: CRAB3 is able to copy but unable to delete directory in %s on site %s. Asynchronous Stage Out with CRAB3 will fail.' % (
colors.RED, colors.NORMAL, self.lfnPrefix,
self.options.sitename)
finalmsg += '\n You may want to contact the site administrators sending them the \'crab checkwrite\' output as printed above.'
returndict = {'status': 'FAILED'}
else:
self.logger.info('\nSuccessfully deleted directory %s' %
(dirpfn))
finalmsg = '%sSuccess%s: Able to write in %s on site %s' % (
colors.GREEN, colors.NORMAL, self.lfnPrefix,
self.options.sitename)
returndict = {'status': 'SUCCESS'}
else:
if 'Permission denied' in cperr or 'mkdir: cannot create directory' in cperr:
finalmsg = '%sError%s: Unable to write in %s on site %s' % (
colors.RED, colors.NORMAL, self.lfnPrefix,
self.options.sitename)
finalmsg += '\n You may want to contact the site administrators sending them the \'crab checkwrite\' output as printed above.'
returndict = {'status': 'FAILED'}
elif 'timeout' in cpout or 'timeout' in cperr:
self.logger.info('Connection time out.')
finalmsg = '\nUnable to check write permission in %s on site %s' % (
self.lfnPrefix, self.options.sitename)
finalmsg += '\nPlease try again later or contact the site administrators sending them the \'crab checkwrite\' output as printed above.'
returndict = {'status': 'FAILED'}
else:
finalmsg = 'Unable to check write permission in %s on site %s' % (
self.lfnPrefix, self.options.sitename)
finalmsg += '\nPlease try again later or contact the site administrators sending them the \'crab checkwrite\' output as printed above.'
returndict = {'status': 'FAILED'}
self.removeFile()
self.logger.info('\nCheckwrite Result:')
self.logger.info(finalmsg)
if returndict['status'] == 'FAILED':
self.logger.info(
'%sNote%s: You cannot write to a site if you did not ask permission.'
% (colors.BOLD, colors.NORMAL))
if 'CH_CERN' in self.options.sitename:
dbgmsg = '%sAdditional diagnostic info for CERN EOS%s\n' % (
colors.RED, colors.NORMAL)
dbgcmd = "echo '== id ==>:';id"
dbgcmd += ";echo '== voms-proxy-info -all ==>:';voms-proxy-info -all"
dbgcmd += ";which uberftp > /dev/null 2>&1 && echo '== uberftp eoscmsftp.cern.ch pwd ==>:'"
dbgcmd += ";which uberftp > /dev/null 2>&1 && uberftp eoscmsftp.cern.ch pwd"
dbgcmd += ";which uberftp > /dev/null 2>&1 || echo 'WARNING uberftp command not found. To get additional diagnostic info'"
dbgcmd += ";which uberftp > /dev/null 2>&1 || echo ' log on lxplus, get a proxy and execute:'"
dbgcmd += ";which uberftp > /dev/null 2>&1 || echo ' uberftp eoscmsftp.cern.ch pwd'"
#self.logger.info('Executing command: %s' % cmd)
#self.logger.info('Please wait...')
output, _, _ = execute_command(command=dbgcmd)
dbgmsg += output
self.logger.info(dbgmsg)
return returndict
def __call__(self):
"""
Copying locally files staged remotely.
*Using a subprocess to encapsulate the copy command.
* maximum parallel download is 10, line 61
* default --sendreceive-timeout is 1800 s, line 75 and 77
"""
globalExitcode = -1
dicttocopy = self.options.inputdict
# taking number of parallel download to create from user, default is 10
if self.options.nparallel == None:
nsubprocess = 10
else:
nsubprocess = int(self.options.nparallel)
if nsubprocess <= 0 or nsubprocess > 20:
self.logger.info(
"Inappropriate number of parallel download, must between 0 to 20 "
)
return -1
command = ""
if cmd_exist("gfal-copy"):
self.logger.info("Will use `gfal-copy` command for file transfers")
command = "env -i gfal-copy -v "
command += " -T "
elif cmd_exist("lcg-cp"):
self.logger.info("Will use `lcg-cp` command for file transfers")
command = "lcg-cp --connect-timeout 20 --verbose -b -D srmv2"
command += " --sendreceive-timeout "
else:
# This should not happen. If it happens, Site Admin have to install GFAL2 (yum install gfal2-util gfal2-all)
self.logger.info(
"%sError%s: Can`t find command `gfal-copy` or `lcg-ls`, Please contact the site administrator."
% (colors.RED, colors.NORMAL))
return [], []
command += "1800" if self.options.waittime == None else str(
1800 + int(self.options.waittime))
# timeout = 20 + 240 + 60 #giving 1 extra minute: 5min20"
srmtimeout = 900 # default transfer timeout in case the file size is unknown: 15min
minsrmtimeout = 60 # timeout cannot be less then 1min
downspeed = float(250 *
1024) # default speed assumes a download of 250KB/s
mindownspeed = 20 * 1024.
manager = Manager()
successfiles = manager.dict()
failedfiles = manager.dict()
self.logger.debug("Starting ChildProcess with %s ChildProcess" %
nsubprocess)
inputq, processarray = self.startchildproc(self.processWorker,
nsubprocess, successfiles,
failedfiles)
for myfile in dicttocopy:
if downspeed < mindownspeed:
downspeed = mindownspeed
fileid = myfile['pfn'].split('/')[-1]
dirpath = os.path.join(
self.options.destination,
myfile['suffix'] if 'suffix' in myfile else '')
url_input = bool(re.match("^[a-z]+://", dirpath))
if not url_input and not os.path.isdir(dirpath):
os.makedirs(dirpath)
localFilename = os.path.join(dirpath, str(fileid))
##### Handling the "already existing file" use case
if not url_input and os.path.isfile(localFilename):
size = os.path.getsize(localFilename)
# delete the file if its size is zero or its size is not the expected size
if size == 0 or ('size' in myfile and myfile['size'] != size):
try:
self.logger.info("Removing %s as it is not complete: current size %s, expected size %s" % (fileid, size, \
myfile['size'] if 'size' in myfile else 'unknown'))
os.remove(localFilename)
except Exception, ex:
self.logger.info(
"%sError%s: Cannot remove the file because of: %s"
% (colors.RED, colors.NORMAL, ex))
# if the file still exists skip it
if not url_input and os.path.isfile(localFilename):
self.logger.info("Skipping %s as file already exists in %s" %
(fileid, localFilename))
continue
##### Creating the command
# timeout based on file size and download speed * 2
maxtime = srmtimeout if not 'size' in myfile or myfile[
'size'] == 0 else int(ceil(2 * myfile['size'] / downspeed))
localsrmtimeout = minsrmtimeout if maxtime < minsrmtimeout else maxtime # do not want a too short timeout
timeout = " --srm-timeout "
if cmd_exist("gfal-copy"):
timeout = " -t "
cmd = '%s %s %s %%s' % (command, timeout + str(localsrmtimeout) +
' ', myfile['pfn'])
if url_input:
cmd = cmd % localFilename
else:
cmd = cmd % ("file://%s" % localFilename)
self.logger.info("Placing file '%s' in retrieval queue " % fileid)
inputq.put((myfile, cmd))