def reconnect(self, attempts=2):
"Reconnect function"
self.username = None
self.client = self._new_client()
att = 0
while att < attempts:
msg = 'Fail to authenticate with %s@%s' \
% (self.username, self.hostname)
msg += '\nPlease try again'
print_error(msg)
try:
username = self._username()
password = self._password(username)
self.client.connect(self.hostname, username=username, \
password=password)
self.username = username
return True
except AuthenticationException:
pass # will retry
except:
msg = 'Unable to connect to %s' % self.hostname
print_error(msg)
raise
att += 1
return False
def get_data(url, kwargs=None, headers=None,
verbose=None, decoder='json', post=False):
"Retrive data"
if not headers and url.find('DBSReader') != -1:
headers = {'Accept': 'application/json' } # DBS3 always needs that
ckey = None
cert = os.path.join(os.environ['HOME'], '.globus/usercert.pem')
try:
# pycurl data look-up, primary way to get the data
mgr = RequestHandler()
with working_pem(PEMMGR.pem) as ckey:
res = mgr.get_data(url, kwargs, headers, post, ckey, cert, verbose=verbose)
if decoder == 'json':
data = json.load(res)
else:
data = res.read()
return data
except Exception as exc:
if verbose:
print_error(exc)
msg = 'Fall back to urllib'
print_warning(msg)
# urllib data look-up, fallback mechanism
with working_pem(PEMMGR.pem) as ckey:
return get_data_helper(url, kwargs, headers,
verbose, decoder, post, ckey, cert)
def cms_rm(arg):
"""
CMS rm command works with local files/dirs and CMS storate elements.
Examples:
cmssh> rm local_file
cmssh> rm -rf local_dir
cmssh> rm T3_US_Cornell:/xrootdfs/cms/store/user/user_name/file.root
"""
arg = arg.strip()
try:
debug = get_ipython().debug
except:
debug = 0
if not arg:
print_error("Usage: rm <options> source_file")
dst = arg.split()[-1]
if os.path.exists(dst) or len(glob.glob(dst)):
cmd = "rm %s" % arg
run(cmd)
else:
if pat_lfn.match(arg.split(':')[-1]):
status = rm_lfn(arg, verbose=debug)
print_status(status)
else:
if not os.path.exists(dst):
print_error('File %s does not exists' % dst)
else:
raise Exception('Not implemented yet')
def cms_rm(arg):
"""
CMS rm command works with local files/dirs and CMS storate elements.
Examples:
cmssh> rm local_file
cmssh> rm -rf local_dir
cmssh> rm T3_US_Cornell:/xrootdfs/cms/store/user/user_name/file.root
"""
arg = arg.strip()
try:
debug = get_ipython().debug
except:
debug = 0
if not arg:
print_error("Usage: rm <options> source_file")
dst = arg.split()[-1]
if os.path.exists(dst) or len(glob.glob(dst)):
cmd = "rm %s" % arg
run(cmd)
else:
if pat_lfn.match(arg.split(':')[-1]):
status = rm_lfn(arg, verbose=debug)
print_status(status)
else:
if not os.path.exists(dst):
print_error('File %s does not exists' % dst)
else:
raise Exception('Not implemented yet')
def reconnect(self, attempts=2):
"Reconnect function"
self.username = None
self.client = self._new_client()
att = 0
while att < attempts:
msg = 'Fail to authenticate with %s@%s' \
% (self.username, self.hostname)
msg += '\nPlease try again'
print_error(msg)
try:
username = self._username()
password = self._password(username)
self.client.connect(self.hostname, username=username, \
password=password)
self.username = username
return True
except AuthenticationException:
pass # will retry
except:
msg = 'Unable to connect to %s' % self.hostname
print_error(msg)
raise
att += 1
return False
def github_issues(arg=None):
"""
Retrieve information about cmssh tickets, e.g.
Examples:
cmssh> tickets # list all cmssh tickets
cmssh> ticket 14 # get details for given ticket id
cmssh> ticket new # post new ticket from cmssh
# or post it at https://github.com/vkuznet/cmssh/issues/new
"""
if arg == 'new':
msg = 'You can post new ticket via web interface at\n'
msg += 'https://github.com/vkuznet/cmssh/issues/new\n'
msg += 'otherwise it will be posted as anonymous gist ticket'
print_info(msg)
if not user_input('Proceed', default='N'):
return
email = raw_input('Your Email : ')
if not email:
msg = "You did your email address"
print_error(msg)
return
desc = ''
msg = 'Type your problem, attach traceback, etc. Once done print '
msg += msg_blue('EOF') + ' and hit ' + msg_blue('Enter') + '\n'
print msg
while True:
try:
uinput = raw_input()
if uinput.strip() == 'EOF':
break
desc += uinput + '\n'
except KeyboardInterrupt:
break
if not desc:
msg = "You did not provide bug description"
print_error(msg)
return
if not user_input('Send this ticket', default='N'):
print_info('Aborting your action')
return
key = 'cmssh-%s' % time.strftime("%Y-%m-%d %H:%M:%S",
time.gmtime(time.time()))
files = {key: {'content': desc}}
res = post_ticket(key, files)
if res.has_key('html_url'):
print_status('New gist ticket %s' % res['html_url'])
title = 'cmssh gist %s' % res['html_url']
if isinstance(res, dict):
ticket = pprint.pformat(res)
else:
ticket = res
to_user = base64.decodestring('dmt1em5ldEBnbWFpbC5jb20=\n')
send_email(to_user, email, title, ticket)
else:
res = get_tickets(arg)
RESMGR.assign(res)
pprint.pprint(res)
def github_issues(arg=None):
"""
Retrieve information about cmssh tickets, e.g.
Examples:
cmssh> tickets # list all cmssh tickets
cmssh> ticket 14 # get details for given ticket id
cmssh> ticket new # post new ticket from cmssh
# or post it at https://github.com/vkuznet/cmssh/issues/new
"""
if arg == 'new':
msg = 'You can post new ticket via web interface at\n'
msg += 'https://github.com/vkuznet/cmssh/issues/new\n'
msg += 'otherwise it will be posted as anonymous gist ticket'
print_info(msg)
if not user_input('Proceed', default='N'):
return
email = raw_input('Your Email : ')
if not email:
msg = "You did your email address"
print_error(msg)
return
desc = ''
msg = 'Type your problem, attach traceback, etc. Once done print '
msg += msg_blue('EOF') + ' and hit ' + msg_blue('Enter') + '\n'
print msg
while True:
try:
uinput = raw_input()
if uinput.strip() == 'EOF':
break
desc += uinput + '\n'
except KeyboardInterrupt:
break
if not desc:
msg = "You did not provide bug description"
print_error(msg)
return
if not user_input('Send this ticket', default='N'):
print_info('Aborting your action')
return
key = 'cmssh-%s' % time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(time.time()))
files = {key: {'content': desc}}
res = post_ticket(key, files)
if res.has_key('html_url'):
print_status('New gist ticket %s' % res['html_url'])
title = 'cmssh gist %s' % res['html_url']
if isinstance(res, dict):
ticket = pprint.pformat(res)
else:
ticket = res
to_user = base64.decodestring('dmt1em5ldEBnbWFpbC5jb20=\n')
send_email(to_user, email, title, ticket)
else:
res = get_tickets(arg)
RESMGR.assign(res)
pprint.pprint(res)
def cms_cp(arg):
"""
cmssh cp command copies local files/dirs to/from local files/dirs or CMS storate elements.
Examples:
cmssh> cp file1 file2
cmssh> cp file.root T3_US_Cornell:/store/user/name
cmssh> cp /store/mc/file.root T3_US_Cornell:/store/user/name
cmssh> cp T3_US_Cornell:/store/user/name/file.root T3_US_Omaha
"""
check_voms_proxy()
background = False
orig_arg = arg
arg = arg.strip()
try:
last_arg = arg.split(' ')[-1].strip()
if last_arg == '&':
background = True
arg = arg.replace('&', '').strip()
src, dst = arg.rsplit(' ', 1)
if dst.find('&') != -1:
background = True
dst = dst.replace('&', '').strip()
if dst == '.':
dst = os.getcwd()
# check if src still has options and user asked for -f
options = src.split(' ')
if len(options) > 1 and options[0] == '-f':
overwrite = True
else:
overwrite = False
except:
traceback.print_exc()
return
try:
debug = get_ipython().debug
except:
debug = 0
if not arg:
print_error("Usage: cp <options> source_file target_{file,directory}")
pat = pat_se
orig = src.split(' ')[-1]
if os.path.exists(orig) and not pat.match(dst):
if background:
cmd = 'cp %s' % orig_arg
subprocess.call(cmd, shell=True)
else:
run("cp %s %s" % (src, dst))
else:
try:
status = copy_lfn(orig, dst, debug, background, overwrite)
print_status(status)
except:
traceback.print_exc()
def check_voms_proxy():
"Check status of user VOMS proxy"
cmd = 'voms-proxy-info -timeleft'
out, err = execmd(cmd)
if err:
print_error('Fail to check user proxy info')
return
if int(out) < 3600: # time left is less then 1 hour
msg = 'Your VOMS proxy will expire in %s sec (< 1 hour). ' % out
msg += 'Please run ' + msg_green('vomsinit') + ' command to renew it'
print_warning(msg)
def transfer(self, method, file1, file2=None):
"Perform sftp transfer action"
if not self.client:
msg = 'Unable to connect to %s@%s' \
% (self.username, self.hostname)
print_error(msg)
return
ftp = self.client.open_sftp()
if not file2:
file2 = file1
getattr(ftp, method)(file1, file2)
ftp.close()
def transfer(self, method, file1, file2=None):
"Perform sftp transfer action"
if not self.client:
msg = 'Unable to connect to %s@%s' \
% (self.username, self.hostname)
print_error(msg)
return
ftp = self.client.open_sftp()
if not file2:
file2 = file1
getattr(ftp, method)(file1, file2)
ftp.close()
def access2file(fname):
"Check if given file name exists on a system and is accessible"
if not os.path.isfile(fname):
msg = 'File %s does not exists' % fname
print_error(msg)
return False
mode = os.stat(fname).st_mode
if not bool(mode & stat.S_IRUSR):
msg = 'Unsufficient privileges to access %s' % fname
print_error(msg)
return False
return True
def check_voms_proxy():
"Check status of user VOMS proxy"
cmd = 'voms-proxy-info -timeleft'
out, err = execmd(cmd)
if err:
print_error('Fail to check user proxy info')
return
if int(out) < 3600: # time left is less then 1 hour
msg = 'Your VOMS proxy will expire in %s sec (< 1 hour). ' % out
msg += 'Please run ' + msg_green('vomsinit') + ' command to renew it'
print_warning(msg)
def access2file(fname):
"Check if given file name exists on a system and is accessible"
if not os.path.isfile(fname):
msg = 'File %s does not exists' % fname
print_error(msg)
return False
mode = os.stat(fname).st_mode
if not bool(mode & stat.S_IRUSR):
msg = 'Unsufficient privileges to access %s' % fname
print_error(msg)
return False
return True
def cms_cp(arg):
"""
cmssh cp command copies local files/dirs to/from local files/dirs or CMS storate elements.
Examples:
cmssh> cp file1 file2
cmssh> cp file.root T3_US_Cornell:/store/user/name
cmssh> cp /store/mc/file.root T3_US_Cornell:/store/user/name
cmssh> cp T3_US_Cornell:/store/user/name/file.root T3_US_Omaha
"""
check_voms_proxy()
background = False
orig_arg = arg
arg = arg.strip()
try:
last_arg = arg.split(' ')[-1].strip()
if last_arg == '&':
background = True
arg = arg.replace('&', '').strip()
src, dst = arg.rsplit(' ', 1)
if dst.find('&') != -1:
background = True
dst = dst.replace('&', '').strip()
if dst == '.':
dst = os.getcwd()
# check if src still has options and user asked for -f
options = src.split(' ')
if len(options) > 1 and options[0] == '-f':
overwrite = True
else:
overwrite = False
except:
traceback.print_exc()
return
try:
debug = get_ipython().debug
except:
debug = 0
if not arg:
print_error("Usage: cp <options> source_file target_{file,directory}")
pat = pat_se
orig = src.split(' ')[-1]
if os.path.exists(orig) and not pat.match(dst):
if background:
cmd = 'cp %s' % orig_arg
subprocess.call(cmd, shell=True)
else:
run("cp %s %s" % (src, dst))
else:
try:
status = copy_lfn(orig, dst, debug, background, overwrite)
print_status(status)
except:
traceback.print_exc()
def debug_http(arg):
"""
Show or set HTTP debug flag. Default is 0.
"""
arg = arg.strip()
if arg:
if arg not in ['0', '1']:
print_error('Please provide 0/1 for debug_http command')
return
print_info("Set HTTP debug level to %s" % arg)
os.environ['HTTPDEBUG'] = arg
else:
print_info("HTTP debug level is %s" % os.environ.get('HTTPDEBUG', 0))
def debug_http(arg):
"""
Show or set HTTP debug flag. Default is 0.
"""
arg = arg.strip()
if arg:
if arg not in ['0', '1']:
print_error('Please provide 0/1 for debug_http command')
return
print_info("Set HTTP debug level to %s" % arg)
os.environ['HTTPDEBUG'] = arg
else:
print_info("HTTP debug level is %s" % os.environ.get('HTTPDEBUG', 0))
def run_lumi_golden_json():
"Get run lumi dict from golden JSON file"
fname = os.environ.get("CMS_JSON", None)
if os.path.isfile(fname):
with open(fname, "r") as json_file:
try:
cms_json = json.load(json_file)
return fname, cms_json
except:
print_error("Unable to decode CMS JSON: %s" % fname)
return fname, {}
else:
msg = "Unable to locate CMS JSON file"
print_warning(msg)
return None, {}
def run_lumi_golden_json():
"Get run lumi dict from golden JSON file"
fname = os.environ.get('CMS_JSON', None)
if os.path.isfile(fname):
with open(fname, 'r') as json_file:
try:
cms_json = json.load(json_file)
return fname, cms_json
except:
print_error('Unable to decode CMS JSON: %s' % fname)
return fname, {}
else:
msg = 'Unable to locate CMS JSON file'
print_warning(msg)
return None, {}
def file_info(lfn, verbose=None):
query = 'find file.name, file.numevents, file.size, file.createdate, file.createby, file.moddate, file.modby where file=%s' % lfn
params = {"api": "executeQuery", "apiversion": "DBS_2_0_9", "query": query}
default_instance = os.environ.get('DBS_INSTANCE')
for inst in dbs_instances():
os.environ['DBS_INSTANCE'] = inst
data = urllib2.urlopen(dbs_url(), urllib.urlencode(params))
try:
rec = [f for f in qlxml_parser(data, 'file')][0]
except:
continue
rec['logical_file_name'] = rec['file']['file.name']
rec['size'] = rec['file']['file.size']
rec['nevents'] = rec['file']['file.numevents']
rec['created'] = time.strftime(
"%Y-%m-%d %H:%M:%S GMT",
time.gmtime(rec['file']['file.createdate']))
rec['createdby'] = rec['file']['file.createby']
rec['modified'] = time.strftime(
"%Y-%m-%d %H:%M:%S GMT", time.gmtime(rec['file']['file.moddate']))
rec['modifiedby'] = rec['file']['file.modby']
del rec['file']
lfnobj = File(rec)
try:
pfnlist, selist = get_pfns(lfn, verbose)
if not selist:
query = 'find site where file=%s' % lfn
params.update({"query": query})
data = urllib2.urlopen(dbs_url(), urllib.urlencode(params))
try:
rec = [f for f in qlxml_parser(data, 'site')][0]
sename = rec['site']['site']
selist = [sename]
pfnlist = lfn2pfn(lfn, sename)
except:
pass
lfnobj.assign('pfn', pfnlist)
lfnobj.assign('se', selist)
except:
traceback.print_exc()
lfnobj.assign('pfn', [])
lfnobj.assign('se', [])
os.environ['DBS_INSTANCE'] = default_instance
lfnobj.assign('dbs_instance', inst)
return lfnobj
os.environ['DBS_INSTANCE'] = default_instance
msg = 'Fail to look-up LFN in %s DBS instances' % dbs_instances()
print_error(msg)
def cms_install(rel):
"""
cmssh command to install given CMSSW release.
Examples:
cmssh> install CMSSW_5_2_4
"""
rel = rel.strip()
pat = pat_release
if not pat.match(rel):
msg = 'Fail to validate release name "%s"' % rel
print_error(msg)
msg = 'Please check the you provide correct release name,'
msg += ' e.g. CMSSW_X_Y_Z<_patchN>'
print msg
return
# check if we have stand-alone installation
if os.environ.get('CMSSH_CMSSW', None):
msg = '\nYou are not allowed to install new release, '
msg += 'since cmssh was installed with system CMSSW install area'
print msg
return
# check if given release/architecture is in place
status = check_release_arch(rel)
if status != 'ok':
msg = '\nCheck release architecture status: %s' % status
print msg
return
print "Searching for %s" % rel
script = get_apt_init(os.environ['SCRAM_ARCH'])
cmd = 'source %s; apt-cache search %s | grep -v -i fwlite' % (script, rel)
run(cmd)
if rel.lower().find('patch') != -1:
print "Installing cms+cmssw-patch+%s ..." % rel
cmd = 'source %s; apt-get install cms+cmssw-patch+%s' % (script, rel)
else:
print "Installing cms+cmssw+%s ..." % rel
cmd = 'source %s; apt-get install cms+cmssw+%s' % (script, rel)
subprocess.call(cmd, shell=True) # use subprocess due to apt-get interactive feature
if platform() == 'osx':
idir = '%s/%s/cms/cmssw/%s' \
% (os.environ['VO_CMS_SW_DIR'], os.environ['SCRAM_ARCH'], rel)
fix_so(idir)
print "Create user area for %s release ..." % rel
cmsrel(rel)
def file_info(lfn, verbose=None):
query = 'find file.name, file.numevents, file.size, file.createdate, file.createby, file.moddate, file.modby where file=%s' % lfn
params = {"api":"executeQuery", "apiversion": "DBS_2_0_9", "query":query}
default_instance = os.environ.get('DBS_INSTANCE')
for inst in dbs_instances():
os.environ['DBS_INSTANCE'] = inst
data = urllib2.urlopen(dbs_url(), urllib.urlencode(params))
try:
rec = [f for f in qlxml_parser(data, 'file')][0]
except:
continue
rec['logical_file_name'] = rec['file']['file.name']
rec['size'] = rec['file']['file.size']
rec['nevents'] = rec['file']['file.numevents']
rec['created'] = time.strftime("%Y-%m-%d %H:%M:%S GMT", time.gmtime(rec['file']['file.createdate']))
rec['createdby'] = rec['file']['file.createby']
rec['modified'] = time.strftime("%Y-%m-%d %H:%M:%S GMT", time.gmtime(rec['file']['file.moddate']))
rec['modifiedby'] = rec['file']['file.modby']
del rec['file']
lfnobj = File(rec)
try:
pfnlist, selist = get_pfns(lfn, verbose)
if not selist:
query = 'find site where file=%s' % lfn
params.update({"query":query})
data = urllib2.urlopen(dbs_url(), urllib.urlencode(params))
try:
rec = [f for f in qlxml_parser(data, 'site')][0]
sename = rec['site']['site']
selist = [sename]
pfnlist = lfn2pfn(lfn, sename)
except:
pass
lfnobj.assign('pfn', pfnlist)
lfnobj.assign('se', selist)
except:
traceback.print_exc()
lfnobj.assign('pfn', [])
lfnobj.assign('se', [])
os.environ['DBS_INSTANCE'] = default_instance
lfnobj.assign('dbs_instance', inst)
return lfnobj
os.environ['DBS_INSTANCE'] = default_instance
msg = 'Fail to look-up LFN in %s DBS instances' % dbs_instances()
print_error(msg)
def cms_install(rel):
"""
cmssh command to install given CMSSW release.
Examples:
cmssh> install CMSSW_5_2_4
"""
rel = rel.strip()
pat = pat_release
if not pat.match(rel):
msg = 'Fail to validate release name "%s"' % rel
print_error(msg)
msg = 'Please check the you provide correct release name,'
msg += ' e.g. CMSSW_X_Y_Z<_patchN>'
print msg
return
# check if we have stand-alone installation
if os.path.islink(os.environ['VO_CMS_SW_DIR']):
msg = '\nYou are not allowed to install new release, '
msg += 'since cmssh was installed with system CMSSW install area'
print msg
return
# check if given release/architecture is in place
status = check_release_arch(rel)
if status != 'ok':
msg = '\nCheck release architecture status: %s' % status
print msg
return
print "Searching for %s" % rel
script = get_apt_init(os.environ['SCRAM_ARCH'])
cmd = 'source %s; apt-cache search %s | grep -v -i fwlite' % (script, rel)
run(cmd)
if rel.lower().find('patch') != -1:
print "Installing cms+cmssw-patch+%s ..." % rel
cmd = 'source %s; apt-get install cms+cmssw-patch+%s' % (script, rel)
else:
print "Installing cms+cmssw+%s ..." % rel
cmd = 'source %s; apt-get install cms+cmssw+%s' % (script, rel)
subprocess.call(
cmd, shell=True) # use subprocess due to apt-get interactive feature
print "Create user area for %s release ..." % rel
cmsrel(rel)
def get_data_helper(url, kwargs=None, headers=None,
verbose=None, decoder='json', post=False, ckey=None, cert=None):
"""Retrieve data helper function"""
if url.find('https') != -1:
if not ckey and not cert:
ckey, cert = get_key_cert()
else:
ckey = None
cert = None
if kwargs:
params = kwargs
else:
params = {}
if url.find('/datasets') != -1: # DBS3 use case
params.update({'dataset_access_type':'PRODUCTION', 'detail':'True'})
encoded_data = urllib.urlencode(params, doseq=True)
if not post:
url = url + '?' + encoded_data
if verbose:
print "Request:", url, encoded_data, headers, ckey, cert
req = urllib2.Request(url)
if headers:
for key, val in headers.items():
req.add_header(key, val)
else:
headers = {'Accept':'application/json;text/json'}
if cert:
handler = HTTPSClientAuthHandler(ckey, cert)
opener = urllib2.build_opener(handler)
urllib2.install_opener(opener)
if post:
print "POST", req, url, encoded_data, params
res = urllib2.urlopen(req, json.dumps(params))
else:
res = urllib2.urlopen(req)
if decoder == 'json':
try:
data = json.load(res)
except Exception as err:
data = {}
print_error(str(err))
else:
data = res.read()
return data
def test_key_cert():
"""Test user key/cert file and their permissions"""
kfile = os.path.join(os.environ['HOME'], '.globus/userkey.pem')
cfile = os.path.join(os.environ['HOME'], '.globus/usercert.pem')
if os.path.isfile(kfile):
if not (check_0600(kfile) or check_0400(kfile)):
msg = "File %s has weak permission settings, try" % kfile
print_warning(msg)
print "chmod 0400 %s" % kfile
else:
print_error("File %s does not exists, grid/cp commands will not work" % kfile)
if os.path.isfile(cfile):
if not (check_0600(cfile) or check_0400(cfile)):
msg = "File %s has weak permission settings, try" % cfile
print_warning(msg)
print "chmod 0600 %s" % cfile
else:
msg = "File %s does not exists, grid/cp commands will not work" % cfile
print_error(msg)
def test_key_cert():
"""Test user key/cert file and their permissions"""
kfile = os.path.join(os.environ['HOME'], '.globus/userkey.pem')
cfile = os.path.join(os.environ['HOME'], '.globus/usercert.pem')
if os.path.isfile(kfile):
if not (check_0600(kfile) or check_0400(kfile)):
msg = "File %s has weak permission settings, try" % kfile
print_warning(msg)
print "chmod 0400 %s" % kfile
else:
print_error("File %s does not exists, grid/cp commands will not work" %
kfile)
if os.path.isfile(cfile):
if not (check_0600(cfile) or check_0400(cfile)):
msg = "File %s has weak permission settings, try" % cfile
print_warning(msg)
print "chmod 0600 %s" % cfile
else:
msg = "File %s does not exists, grid/cp commands will not work" % cfile
print_error(msg)
def run_lumi_info(arg, verbose=None):
"Get int. luminosity for given argument (can be dataset, block, file, run)"
run_lumi = run_lumi_dict(arg, verbose)
if not run_lumi:
print_error("Empty run-lumi list")
return []
totlumi, lumiunit = lumidb(run_lumi_dict=run_lumi, lumi_report=verbose)
print "Delivered luminosity %s (%s)" % (totlumi, lumiunit)
if verbose:
print "Input run lumi dict", pprint.pprint(run_lumi)
golden_fname, golden_json = run_lumi_golden_json()
if golden_json:
if verbose:
print "Intersect with CMS JSON:", golden_fname
rdict = run_lumi_subset(golden_json, run_lumi)
totlumi, lumiunit = lumidb(rdict, lumi_report=verbose)
print "Delivered luminosity wrt CMS JSON: %s (%s)" % (totlumi, lumiunit)
if verbose:
print "Intersected run lumi dict", pprint.pprint(rdict)
return []
def run_lumi_info(arg, verbose=None):
"Get int. luminosity for given argument (can be dataset, block, file, run)"
run_lumi = run_lumi_dict(arg, verbose)
if not run_lumi:
print_error('Empty run-lumi list')
return []
totlumi, lumiunit = lumidb(run_lumi_dict=run_lumi, lumi_report=verbose)
print "Delivered luminosity %s (%s)" % (totlumi, lumiunit)
if verbose:
print "Input run lumi dict", pprint.pprint(run_lumi)
golden_fname, golden_json = run_lumi_golden_json()
if golden_json:
if verbose:
print "Intersect with CMS JSON:", golden_fname
rdict = run_lumi_subset(golden_json, run_lumi)
totlumi, lumiunit = lumidb(rdict, lumi_report=verbose)
print "Delivered luminosity wrt CMS JSON: %s (%s)" % (totlumi,
lumiunit)
if verbose:
print "Intersected run lumi dict", pprint.pprint(rdict)
return []
def get_size(surl, verbose=None):
"""
Execute srm-ls <surl> command and retrieve file size information
"""
srmls = os.environ.get('SRM_LS', '')
if not srmls:
print_error('Unable to find srm ls tool')
sys.exit(1)
if srmls.find('srm-ls') != -1:
srmargs = ''
else:
srmargs = '-2'
cmd = '%s %s %s' % (srmls, srmargs, surl)
if verbose:
print_info(cmd)
if cmd.find('file:///') != -1:
return file_size(cmd.split('file:///')[-1])
stdout, stderr = execmd(cmd)
if verbose:
print_info(stdout + stderr)
orig_size = 0
if cmd.find('file:///') != -1: # srm-ls returns XML
if srmls.find('srm-ls') != -1:
orig_size = parse_srmls(stdout)
else:
try:
orig_size = stdout.split()[0].strip()
except:
return 0
else:
if srmls.find('srm-ls') != -1:
for line in stdout.split('\n'):
if line.find('Bytes') != -1:
orig_size = line.replace('\n', '').split('=')[-1]
else:
try:
orig_size = stdout.split()[0].strip()
except:
return 0
return orig_size
def list_se(self, arg, verbose=0):
"""list content of given directory on SE"""
try:
node, ldir = arg.split(':')
except:
msg = 'Given argument "%s" does not represent SE:dir' % arg
raise Exception(msg)
srmls = os.environ.get('SRM_LS', '')
if not srmls:
print_error('Unable to find srm ls tool')
sys.exit(1)
dst = [r for r in resolve_user_srm_path(node, ldir)][0]
if os.environ.get('LCG_LS', ''):
cmd = "%s -l -v -b -D srmv2 %s" % (os.environ['LCG_LS'], dst)
else:
if srmls.find('srm-ls') != -1:
cmd = "%s %s -fulldetailed" % (srmls, dst)
else:
cmd = "%s -2 -l %s" % (srmls, dst)
if verbose:
print cmd
stdout, stderr = execmd(cmd)
if stderr:
print_error(stderr)
output = []
row = {}
if os.environ.get('LCG_LS', ''):
for line in stdout.split('\n'):
if line.find('SE type') != -1:
continue
output.append(line)
return '\n'.join(output)
elif srmls.find('srmls') != -1:
for line in srmls_printer(stdout, dst.split('=')[-1]):
output.append(line)
return '\n'.join(output)
else:
for line in srm_ls_printer(stdout, dst.split('=')[-1]):
output.append(line)
return '\n'.join(output)
def list_se(self, arg, verbose=0):
"""list content of given directory on SE"""
try:
node, ldir = arg.split(':')
except:
msg = 'Given argument "%s" does not represent SE:dir' % arg
raise Exception(msg)
srmls = os.environ.get('SRM_LS', '')
if not srmls:
print_error('Unable to find srm ls tool')
sys.exit(1)
dst = [r for r in resolve_user_srm_path(node, ldir)][0]
if os.environ.get('LCG_LS', ''):
cmd = "%s -l -v -b -D srmv2 %s" % (os.environ['LCG_LS'], dst)
else:
if srmls.find('srm-ls') != -1:
cmd = "%s %s -fulldetailed" % (srmls, dst)
else:
cmd = "%s -2 -l %s" % (srmls, dst)
if verbose:
print cmd
stdout, stderr = execmd(cmd)
if stderr:
print_error(stderr)
output = []
row = {}
if os.environ.get('LCG_LS', ''):
for line in stdout.split('\n'):
if line.find('SE type') != -1:
continue
output.append(line)
return '\n'.join(output)
elif srmls.find('srmls') != -1:
for line in srmls_printer(stdout, dst.split('=')[-1]):
output.append(line)
return '\n'.join(output)
else:
for line in srm_ls_printer(stdout, dst.split('=')[-1]):
output.append(line)
return '\n'.join(output)
def get_size(surl, verbose=None):
"""
Execute srm-ls <surl> command and retrieve file size information
"""
srmls = os.environ.get('SRM_LS', '')
if not srmls:
print_error('Unable to find srm ls tool')
sys.exit(1)
if srmls.find('srm-ls') != -1:
srmargs = ''
else:
srmargs = '-2'
cmd = '%s %s %s' % (srmls, srmargs, surl)
if verbose:
print_info(cmd)
if cmd.find('file:///') != -1:
return file_size(cmd.split('file:///')[-1])
stdout, stderr = execmd(cmd)
if verbose:
print_info(stdout + stderr)
orig_size = 0
if cmd.find('file:///') != -1: # srm-ls returns XML
if srmls.find('srm-ls') != -1:
orig_size = parse_srmls(stdout)
else:
try:
orig_size = stdout.split()[0].strip()
except:
return 0
else:
if srmls.find('srm-ls') != -1:
for line in stdout.split('\n'):
if line.find('Bytes') != -1:
orig_size = line.replace('\n', '').split('=')[-1]
else:
try:
orig_size = stdout.split()[0].strip()
except:
return 0
return orig_size
def cmscrab(arg):
"""
Execute CRAB command, help is available at
https://twiki.cern.ch/twiki/bin/view/CMSPublic/SWGuideCrabFaq
"""
msg = \
'CRAB FAQ: https://twiki.cern.ch/twiki/bin/view/CMSPublic/SWGuideCrabFaq'
print_info(msg)
# check if release version and work area are set (should be set at cmsrel)
rel = os.environ.get('CMSSW_VERSION', None)
work_area = os.environ.get('CMSSW_WORKAREA', None)
if not rel or not work_area:
msg = 'In order to run crab command you must '
msg += 'run ' + msg_blue('cmsrel') + ' command'
print_error(msg)
return
# check existence of crab.cfg
crab_dir = os.path.join(work_area, 'crab')
crab_cfg = os.path.join(crab_dir, 'crab.cfg')
if not os.path.isdir(crab_dir):
os.makedirs(crab_dir)
os.chdir(crab_dir)
if not os.path.isfile(crab_cfg):
msg = 'No crab.cfg file found in %s' % crab_dir
print_warning(msg)
msg = 'Would you like to create one'
if user_input(msg, default='N'):
with open('crab.cfg', 'w') as config:
config.write(crabconfig())
msg = 'Your crab.cfg has been created, please edit it '
msg += 'appropriately and re-run crab command'
print_info(msg)
print "cwd:", os.getcwd()
return
if os.uname()[0] == 'Darwin' and arg == '-submit':
crab_submit_remotely(rel, work_area)
return
cmd = 'source $CRAB_ROOT/crab.sh; crab %s' % arg
cmsexe(cmd)
def cms_mkdir(arg):
"""
cmssh mkdir command creates directory on local filesystem or remote CMS storage element.
Examples:
cmssh> mkdir foo
cmssh> mkdir T3_US_Cornell:/store/user/user_name/foo
"""
arg = arg.strip()
try:
debug = get_ipython().debug
except:
debug = 0
if not arg:
print_error("Usage: mkdir <options> dir")
if arg.find(':') == -1: # not a SE:dir pattern
run("mkdir %s" % arg)
else:
try:
status = mkdir(arg, verbose=debug)
print_status(status)
except:
traceback.print_exc()
def cms_rmdir(arg):
"""
cmssh rmdir command removes directory from local file system or CMS storage element.
Examples:
cmssh> rmdir foo
cmssh> rmdir T3_US_Cornell:/store/user/user_name/foo
"""
arg = arg.strip()
try:
debug = get_ipython().debug
except:
debug = 0
if not arg:
print_error("Usage: rmdir <options> dir")
if os.path.exists(arg):
run("rmdir %s" % arg)
else:
try:
status = rmdir(arg, verbose=debug)
print_status(status)
except:
traceback.print_exc()
def cms_rmdir(arg):
"""
cmssh rmdir command removes directory from local file system or CMS storage element.
Examples:
cmssh> rmdir foo
cmssh> rmdir T3_US_Cornell:/store/user/user_name/foo
"""
arg = arg.strip()
try:
debug = get_ipython().debug
except:
debug = 0
if not arg:
print_error("Usage: rmdir <options> dir")
if os.path.exists(arg):
run("rmdir %s" % arg)
else:
try:
status = rmdir(arg, verbose=debug)
print_status(status)
except:
traceback.print_exc()
def cms_mkdir(arg):
"""
cmssh mkdir command creates directory on local filesystem or remote CMS storage element.
Examples:
cmssh> mkdir foo
cmssh> mkdir T3_US_Cornell:/store/user/user_name/foo
"""
arg = arg.strip()
try:
debug = get_ipython().debug
except:
debug = 0
if not arg:
print_error("Usage: mkdir <options> dir")
if arg.find(':') == -1: # not a SE:dir pattern
run("mkdir %s" % arg)
else:
try:
status = mkdir(arg, verbose=debug)
print_status(status)
except:
traceback.print_exc()
def cmscrab(arg):
"""
Execute CRAB command, help is available at
https://twiki.cern.ch/twiki/bin/view/CMSPublic/SWGuideCrabFaq
"""
msg = \
'CRAB FAQ: https://twiki.cern.ch/twiki/bin/view/CMSPublic/SWGuideCrabFaq'
print_info(msg)
# check if release version and work area are set (should be set at cmsrel)
rel = os.environ.get('CMSSW_VERSION', None)
work_area = os.environ.get('CMSSW_WORKAREA', None)
if not rel or not work_area:
msg = 'In order to run crab command you must '
msg += 'run ' + msg_blue('cmsrel') + ' command'
print_error(msg)
return
# check existence of crab.cfg
crab_dir = os.path.join(work_area, 'crab')
crab_cfg = os.path.join(crab_dir, 'crab.cfg')
if not os.path.isdir(crab_dir):
os.makedirs(crab_dir)
os.chdir(crab_dir)
if not os.path.isfile(crab_cfg):
msg = 'No crab.cfg file found in %s' % crab_dir
print_warning(msg)
msg = 'Would you like to create one'
if user_input(msg, default='N'):
with open('crab.cfg', 'w') as config:
config.write(crabconfig())
msg = 'Your crab.cfg has been created, please edit it '
msg += 'appropriately and re-run crab command'
print_info(msg)
print "cwd:", os.getcwd()
return
if os.uname()[0] == 'Darwin' and arg == '-submit':
crab_submit_remotely(rel, work_area)
return
cmd = 'source $CRAB_ROOT/crab.sh; crab %s' % arg
cmsexe(cmd)
def copy(self, lfn, dst, method='xrdcp', verbose=0, background=False):
"""Copy LFN to given destination"""
if method not in self.methods:
print_error('Unknown transfer method "%s"' % method)
return 'fail'
for xrdcmd, lcgcmd, srmcmd, pfn, pdst in self.transfer_cmds(lfn, dst, verbose):
if method == 'xrdcp':
cmd = xrdcmd
elif method == 'lcgcp':
cmd = lcgcmd
else:
cmd = srmcmd
if not cmd:
return 'fail'
if background:
# I need to pass list of commands for transfer method
# for that I'll use background variable
background = [xrdcmd, lcgcmd, srmcmd]
status = self.transfer(cmd, lfn, pfn, pdst, verbose, background)
if status == 'success' or status == 'accepted':
return status
return 'fail'
def copy(self, lfn, dst, method='xrdcp', verbose=0, background=False):
"""Copy LFN to given destination"""
if method not in self.methods:
print_error('Unknown transfer method "%s"' % method)
return 'fail'
for xrdcmd, lcgcmd, srmcmd, pfn, pdst in self.transfer_cmds(
lfn, dst, verbose):
if method == 'xrdcp':
cmd = xrdcmd
elif method == 'lcgcp':
cmd = lcgcmd
else:
cmd = srmcmd
if not cmd:
return 'fail'
if background:
# I need to pass list of commands for transfer method
# for that I'll use background variable
background = [xrdcmd, lcgcmd, srmcmd]
status = self.transfer(cmd, lfn, pfn, pdst, verbose, background)
if status == 'success' or status == 'accepted':
return status
return 'fail'
def check_permission(dst, verbose=None):
"""
Check permission to write to given destination area
"""
if verbose:
print "Check permission to write to %s" % dst
srmmkdir = os.environ.get('SRM_MKDIR', '')
if not srmmkdir:
print_error('Unable to find srm mkdir command')
sys.exit(1)
cmd = '%s %s' % (srmmkdir, dst)
stdout, stderr = execmd(cmd)
if stderr.find('command not found') != -1:
print 'Unable to find srm mkdir tool'
print help
sys.exit(1)
if stdout.find('SRM-DIR: directory not created') != -1 or\
stdout.find('SRM_FAILURE') != -1:
msg = "Unable to access %s:" % dst
print msg
print "-" * len(msg)
print
print stdout
sys.exit(1)
def get_data(url,
kwargs=None,
headers=None,
verbose=None,
decoder='json',
post=False):
"Retrive data"
if not headers and url.find('DBSReader') != -1:
headers = {'Accept': 'application/json'} # DBS3 always needs that
ckey = None
cert = os.path.join(os.environ['HOME'], '.globus/usercert.pem')
try:
# pycurl data look-up, primary way to get the data
mgr = RequestHandler()
with working_pem(PEMMGR.pem) as ckey:
res = mgr.get_data(url,
kwargs,
headers,
post,
ckey,
cert,
verbose=verbose)
if decoder == 'json':
data = json.load(res)
else:
data = res.read()
return data
except Exception as exc:
if verbose:
print_error(exc)
msg = 'Fall back to urllib'
print_warning(msg)
# urllib data look-up, fallback mechanism
with working_pem(PEMMGR.pem) as ckey:
return get_data_helper(url, kwargs, headers, verbose, decoder,
post, ckey, cert)
def check_permission(dst, verbose=None):
"""
Check permission to write to given destination area
"""
if verbose:
print "Check permission to write to %s" % dst
srmmkdir = os.environ.get('SRM_MKDIR', '')
if not srmmkdir:
print_error('Unable to find srm mkdir command')
sys.exit(1)
cmd = '%s %s' % (srmmkdir, dst)
stdout, stderr = execmd(cmd)
if stderr.find('command not found') != -1:
print 'Unable to find srm mkdir tool'
print help
sys.exit(1)
if stdout.find('SRM-DIR: directory not created') != -1 or\
stdout.find('SRM_FAILURE') != -1:
msg = "Unable to access %s:" % dst
print msg
print "-" * len(msg)
print
print stdout
sys.exit(1)
def cms_ls(arg):
"""
cmssh ls command lists local files/dirs/CMS storate elements or
CMS entities (se, site, dataset, block, run, release, file).
Examples:
cmssh> ls # UNIX command
cmssh> ls -l local_file
cmssh> ls T3_US_Cornell:/store/user/valya
cmssh> ls run=160915
"""
arg = arg.strip()
res = []
try:
debug = get_ipython().debug
except:
debug = 0
orig_arg = arg
if orig_arg.find('|') != -1:
arg, flt = orig_arg.split('|', 1)
arg = arg.strip()
else:
flt = None
startswith = None
entities = \
['se', 'site', 'lfn', 'dataset', 'block', 'run', 'release', 'file']
for item in entities:
if arg.startswith(item + '='):
startswith = item
if os.path.isfile(orig_arg) or os.path.isdir(orig_arg):
cmd = 'ls ' + orig_arg
run(cmd, shell=True)
elif pat_se.match(arg):
arg = arg.replace('site=', '')
res = list_se(arg, debug)
elif pat_site.match(arg):
arg = arg.replace('site=', '')
res = site_info(arg, debug)
elif pat_lfn.match(arg):
arg = arg.replace('file=', '')
arg = arg.replace('lfn=', '')
res = file_info(arg, debug)
elif pat_block.match(arg):
arg = arg.replace('block=', '')
res = block_info(arg, debug)
elif pat_dataset.match(arg):
arg = arg.replace('dataset=', '')
try:
res = dataset_info(arg, debug)
except IndexError:
msg = "Given pattern '%s' does not exist on local filesystem or in DBS" % arg
print_error(msg)
elif pat_run.match(arg):
arg = arg.replace('run=', '')
res = run_info(arg, debug)
elif pat_release.match(arg):
arg = arg.replace('release=', '')
res = release_info(arg, debug)
elif startswith:
msg = 'No pattern is allowed for %s look-up' % startswith
print_error(msg)
else:
cmd = 'ls ' + orig_arg
run(cmd, shell=True)
if res:
RESMGR.assign(res)
list_results(res, debug=True, flt=flt)
def cms_ls(arg):
"""
cmssh ls command lists local files/dirs/CMS storate elements or
CMS entities (se, site, dataset, block, run, release, file).
Examples:
cmssh> ls # UNIX command
cmssh> ls -l local_file
cmssh> ls T3_US_Cornell:/store/user/valya
cmssh> ls run=160915
"""
arg = arg.strip()
res = []
try:
debug = get_ipython().debug
except:
debug = 0
orig_arg = arg
if orig_arg.find('|') != -1:
arg, flt = orig_arg.split('|', 1)
arg = arg.strip()
else:
flt = None
startswith = None
entities = \
['se', 'site', 'lfn', 'dataset', 'block', 'run', 'release', 'file']
for item in entities:
if arg.startswith(item + '='):
startswith = item
if os.path.isfile(orig_arg) or os.path.isdir(orig_arg):
cmd = 'ls ' + orig_arg
run(cmd, shell=True)
elif pat_se.match(arg):
arg = arg.replace('site=', '')
res = list_se(arg, debug)
elif pat_site.match(arg):
arg = arg.replace('site=', '')
res = site_info(arg, debug)
elif pat_lfn.match(arg):
arg = arg.replace('file=', '')
arg = arg.replace('lfn=', '')
res = file_info(arg, debug)
elif pat_block.match(arg):
arg = arg.replace('block=', '')
res = block_info(arg, debug)
elif pat_dataset.match(arg):
arg = arg.replace('dataset=', '')
try:
res = dataset_info(arg, debug)
except IndexError:
msg = "Given pattern '%s' does not exist on local filesystem or in DBS" % arg
print_error(msg)
elif pat_run.match(arg):
arg = arg.replace('run=', '')
res = run_info(arg, debug)
elif pat_release.match(arg):
arg = arg.replace('release=', '')
res = release_info(arg, debug)
elif startswith:
msg = 'No pattern is allowed for %s look-up' % startswith
print_error(msg)
else:
cmd = 'ls ' + orig_arg
run(cmd, shell=True)
if res:
RESMGR.assign(res)
list_results(res, debug=True, flt=flt)
def run(cmd, cdir=None, log=None, msg=None, debug=None, shell=False, call=False):
"Run given command via subprocess call"
if msg:
print msg
cmd = cmd.strip()
kwds = {}
if shell or cmd.find(';') != -1 or cmd.find('&&') != -1 or \
cmd.find('|') != -1:
kwds = {'shell': True}
else:
if isinstance(cmd, unicode):
cmd = shlex.split(cmd.encode('ascii', 'ignore'))
else:
cmd = shlex.split(cmd)
msg = 'execute cmd=%s, kwds=%s' % (cmd, kwds)
if debug:
print_info(msg.capitalize())
try:
with working_dir(cdir):
if not call:
kwds.update({'stdout':subprocess.PIPE,
'stderr':subprocess.PIPE,
'close_fds':True})
if log:
with open(log, 'w') as logstream:
kwds.update({'stdout': logstream, 'stderr': logstream})
if call:
subprocess.call(cmd, **kwds)
return
pipe = subprocess.Popen(cmd, **kwds)
else:
if call:
subprocess.call(cmd, **kwds)
return
pipe = subprocess.Popen(cmd, **kwds)
(child_stdout, child_stderr) = (pipe.stdout, pipe.stderr)
if child_stdout:
stdout = child_stdout.read()
else:
stdout = ''
if child_stderr:
stderr = child_stderr.read()
else:
stderr = ''
if stderr:
if isinstance(cmd, list):
cmd_str = ' '.join(cmd)
else:
cmd_str = cmd
if cmd_str.find('proxy') != -1: # proxy commands prints . to stderr
sss = stderr.replace(stderr[0], '')
if not (stderr[0] == '.' and not len(sss)):
print_error(stderr)
else:
print_error(stderr)
if stdout:
print stdout
except OSError as err:
msg = 'OSError, fail to ' + msg + ', error=%s' % str(err)
print_error(msg)
except Exception as err:
msg = 'Fail to ' + msg + ', error=%s' % str(err)
print_error(msg)
def run(cmd,
cdir=None,
log=None,
msg=None,
debug=None,
shell=False,
call=False):
"Run given command via subprocess call"
if msg:
print msg
cmd = cmd.strip()
kwds = {}
if shell or cmd.find(';') != -1 or cmd.find('&&') != -1 or \
cmd.find('|') != -1:
kwds = {'shell': True}
else:
if isinstance(cmd, unicode):
cmd = shlex.split(cmd.encode('ascii', 'ignore'))
else:
cmd = shlex.split(cmd)
msg = 'execute cmd=%s, kwds=%s' % (cmd, kwds)
if debug:
print_info(msg.capitalize())
try:
with working_dir(cdir):
if not call:
kwds.update({
'stdout': subprocess.PIPE,
'stderr': subprocess.PIPE,
'close_fds': True
})
if log:
with open(log, 'w') as logstream:
kwds.update({'stdout': logstream, 'stderr': logstream})
if call:
subprocess.call(cmd, **kwds)
return
pipe = subprocess.Popen(cmd, **kwds)
else:
if call:
subprocess.call(cmd, **kwds)
return
pipe = subprocess.Popen(cmd, **kwds)
(child_stdout, child_stderr) = (pipe.stdout, pipe.stderr)
if child_stdout:
stdout = child_stdout.read()
else:
stdout = ''
if child_stderr:
stderr = child_stderr.read()
else:
stderr = ''
if stderr:
if isinstance(cmd, list):
cmd_str = ' '.join(cmd)
else:
cmd_str = cmd
if cmd_str.find(
'proxy') != -1: # proxy commands prints . to stderr
sss = stderr.replace(stderr[0], '')
if not (stderr[0] == '.' and not len(sss)):
print_error(stderr)
else:
print_error(stderr)
if stdout:
print stdout
except OSError as err:
msg = 'OSError, fail to ' + msg + ', error=%s' % str(err)
print_error(msg)
except Exception as err:
msg = 'Fail to ' + msg + ', error=%s' % str(err)
print_error(msg)
def transfer(self, cmd, lfn, pfn, pdst, verbose=0, background=False):
"""Copy LFN to given destination"""
err = 'Unable to identify total size of the file,'
err += ' GRID middleware fails.'
if not background:
bar = PrintProgress('Fetching LFN info')
if verbose:
print_info(cmd)
if background:
# here background is a list of commands
if not isinstance(background, list):
return 'fail'
proc = Process(target=execute, args=(background, pfn, pdst, 0))
self.queue[lfn] = (proc, None)
return 'accepted'
elif verbose:
status = execute(cmd, pfn, pdst, verbose)
if not status:
return 'fail'
else:
dst, dst_size = status
size = size_format(dst_size)
if not size or not dst_size:
print_error(err)
print "Status of transfer:\n", status
return 'fail'
else:
print "\nDone, file located at %s (%s)" \
% (dst, size_format(dst_size))
return 'success'
else:
ifile = pdst
pfn_size = get_size(pfn)
if pfn_size and pfn_size != 'null':
tot_size = float(pfn_size)
bar.print_msg('LFN size=%s' % size_format(tot_size))
bar.init('Download in progress:')
proc = Process(target=execute, args=(cmd, pfn, pdst, verbose))
proc.start()
while True:
if proc.is_alive():
size = get_size(ifile)
if not size or size == 'null':
bar.refresh('')
pass
else:
progress = float(size) * 100 / tot_size
bar.refresh(progress)
if progress == 100:
break
else:
break
time.sleep(0.5)
bar.clear()
status = check_file(pfn, pdst, verbose)
if status:
return 'success'
else:
print_error(err)
return 'fail'
return 'fail'
def transfer(self, cmd, lfn, pfn, pdst, verbose=0, background=False):
"""Copy LFN to given destination"""
err = 'Unable to identify total size of the file,'
err += ' GRID middleware fails.'
if not background:
bar = PrintProgress('Fetching LFN info')
if verbose:
print_info(cmd)
if background:
# here background is a list of commands
if not isinstance(background, list):
return 'fail'
proc = Process(target=execute, args=(background, pfn, pdst, 0))
self.queue[lfn] = (proc, None)
return 'accepted'
elif verbose:
status = execute(cmd, pfn, pdst, verbose)
if not status:
return 'fail'
else:
dst, dst_size = status
size = size_format(dst_size)
if not size or not dst_size:
print_error(err)
print "Status of transfer:\n", status
return 'fail'
else:
print "\nDone, file located at %s (%s)" \
% (dst, size_format(dst_size))
return 'success'
else:
ifile = pdst
pfn_size = get_size(pfn)
if pfn_size and pfn_size != 'null':
tot_size = float(pfn_size)
bar.print_msg('LFN size=%s' % size_format(tot_size))
bar.init('Download in progress:')
proc = Process(target=execute, args=(cmd, pfn, pdst, verbose))
proc.start()
while True:
if proc.is_alive():
size = get_size(ifile)
if not size or size == 'null':
bar.refresh('')
pass
else:
progress = float(size)*100/tot_size
bar.refresh(progress)
if progress == 100:
break
else:
break
time.sleep(0.5)
bar.clear()
status = check_file(pfn, pdst, verbose)
if status:
return 'success'
else:
print_error(err)
return 'fail'
return 'fail'