def integration_tests(_arg):
"Run series of integration tests for cmssh"
for fname in ['file1.root', 'file2.root']:
if os.path.isfile(fname):
os.remove(fname)
lfn = \
'/store/data/CRUZET3/Cosmics/RAW/v1/000/050/832/186585EC-024D-DD11-B747-000423D94AA8.root'
lfn2 = \
'/store/data/CRUZET3/Cosmics/RAW/v1/000/050/796/4E1D3610-E64C-DD11-8629-001D09F251FE.root'
dataset = '/PhotonHad/Run2011A-PromptReco-v1/RECO'
dataset2 = '/SUSY_LM9_sftsht_8TeV-pythia6/Summer12-START50_V13-v1/GEN-SIM'
run = 160915
sename = 'T3_US_Cornell:/store/user/valya'
cmd_list = ['pager 0', 'debug_http 0']
cmd_list += ['ls', 'mkdir ttt', 'ls -l', 'rmdir ttt', 'ls']
cmd_list += [
'ls dataset=%s' % dataset,
'ls run=%s' % run,
'ls file=%s' % lfn
]
cmd_list += ['ls %s' % dataset, 'info %s' % dataset]
cmd_list += ['find dataset=/ZMM*', 'das dataset=/ZMM*']
cmd_list += [
'find lumi dataset=%s' % dataset, 'find lumi {"190704":[1,2,3]}',
'find lumi {190704:[1,2,3]}'
]
cmd_list += ['find config dataset=%s' % dataset2]
cmd_list += ['du T3_US_Cornell', 'ls T3_US_Cornell']
cmd_list += [
'ls %s' % sename,
'mkdir %s/foo' % sename,
'ls %s' % sename,
'rmdir %s/foo' % sename,
'ls %s' % sename,
]
cmd_list += [
'cp %s file.root' % lfn, 'ls',
'cp file.root %s' % sename,
'ls %s' % sename,
'rm %s/file.root' % sename,
'ls %s' % sename, 'rm file.root',
'cp %s file1.root &' % lfn,
'cp %s file2.root &' % lfn2, 'ls'
]
cmd_list += ['find user=oliver', 'jobs list', 'jobs user=AikenOliver']
cmd_list += ['releases list', 'arch list', 'jobs', 'ls']
cmd_list += [
'read https://twiki.cern.ch/twiki/bin/viewauth/CMS/SWGuideLHEtoEOS'
]
mgr = get_ipython()
for item in cmd_list:
print_info("Execute %s" % item)
split = item.split(' ', 1)
if len(split) == 1:
cmd = item
args = ''
else:
cmd = split[0]
args = split[-1]
mgr.run_line_magic(cmd, args)
def execute(cmds, src, dst, verbose):
"""
Execute given command, but also check if file is in place at dst
"""
status = check_file(src, dst, verbose)
if status:
return status
else:
if isinstance(cmds, basestring):
stdout, stderr = execmd(cmds)
if verbose:
print_info('Output of %s' % cmd)
print stdout + stderr
status = check_file(src, dst, verbose)
elif isinstance(cmds, list):
for cmd in cmds:
if not cmd:
continue
stdout, stderr = execmd(cmd)
if verbose:
print_info('Output of %s' % cmd)
print stdout + stderr
status = check_file(src, dst, verbose)
if status:
return status
return status
def execute(cmds, src, dst, verbose):
"""
Execute given command, but also check if file is in place at dst
"""
status = check_file(src, dst, verbose)
if status:
return status
else:
if isinstance(cmds, basestring):
stdout, stderr = execmd(cmds)
if verbose:
print_info('Output of %s' % cmd)
print stdout + stderr
status = check_file(src, dst, verbose)
elif isinstance(cmds, list):
for cmd in cmds:
if not cmd:
continue
stdout, stderr = execmd(cmd)
if verbose:
print_info('Output of %s' % cmd)
print stdout + stderr
status = check_file(src, dst, verbose)
if status:
return status
return status
def rm_lfn(self, arg, verbose=0):
"""Remove user lfn from a node"""
try:
node, lfn = arg.split(':')
except:
msg = 'Given argument "%s" does not represent SE:LFN' % arg
raise Exception(msg)
cmd = os.environ.get('SRM_RM', '')
dst = [r for r in resolve_user_srm_path(node)][0]
dst, path = dst.split('=')
if dst[-1] != '=':
dst += '='
for item in lfn.split('/'):
if not item or item in path:
continue
path += '/%s' % item
cmd = "%s %s" % (cmd, dst + path)
if verbose:
print cmd
try:
stdout, stderr = execmd(cmd)
if verbose:
print_info(stdout + stderr)
except:
return 'fail'
return 'success'
def rm_lfn(self, arg, verbose=0):
"""Remove user lfn from a node"""
try:
node, lfn = arg.split(':')
except:
msg = 'Given argument "%s" does not represent SE:LFN' % arg
raise Exception(msg)
cmd = os.environ.get('SRM_RM', '')
dst = [r for r in resolve_user_srm_path(node)][0]
dst, path = dst.split('=')
if dst[-1] != '=':
dst += '='
for item in lfn.split('/'):
if not item or item in path:
continue
path += '/%s' % item
cmd = "%s %s" % (cmd, dst+path)
if verbose:
print cmd
try:
stdout, stderr = execmd(cmd)
if verbose:
print_info(stdout + stderr)
except:
return 'fail'
return 'success'
def cmsenv(_arg):
"cmsenv command"
# in CMS cmsenv is an alias to: eval `scramv1 runtime -sh`'
msg = 'Within cmssh it is not required to use cmsenv\n'
msg += 'please use ' + msg_green('cmsrel') + ' command and '
msg += 'CMS release environment will be set for you'
print_info(msg)
def github_issues(arg=None):
"""
Retrieve information about cmssh tickets, e.g.
Examples:
cmssh> tickets # list all cmssh tickets
cmssh> ticket 14 # get details for given ticket id
cmssh> ticket new # post new ticket from cmssh
# or post it at https://github.com/vkuznet/cmssh/issues/new
"""
if arg == 'new':
msg = 'You can post new ticket via web interface at\n'
msg += 'https://github.com/vkuznet/cmssh/issues/new\n'
msg += 'otherwise it will be posted as anonymous gist ticket'
print_info(msg)
if not user_input('Proceed', default='N'):
return
email = raw_input('Your Email : ')
if not email:
msg = "You did your email address"
print_error(msg)
return
desc = ''
msg = 'Type your problem, attach traceback, etc. Once done print '
msg += msg_blue('EOF') + ' and hit ' + msg_blue('Enter') + '\n'
print msg
while True:
try:
uinput = raw_input()
if uinput.strip() == 'EOF':
break
desc += uinput + '\n'
except KeyboardInterrupt:
break
if not desc:
msg = "You did not provide bug description"
print_error(msg)
return
if not user_input('Send this ticket', default='N'):
print_info('Aborting your action')
return
key = 'cmssh-%s' % time.strftime("%Y-%m-%d %H:%M:%S",
time.gmtime(time.time()))
files = {key: {'content': desc}}
res = post_ticket(key, files)
if res.has_key('html_url'):
print_status('New gist ticket %s' % res['html_url'])
title = 'cmssh gist %s' % res['html_url']
if isinstance(res, dict):
ticket = pprint.pformat(res)
else:
ticket = res
to_user = base64.decodestring('dmt1em5ldEBnbWFpbC5jb20=\n')
send_email(to_user, email, title, ticket)
else:
res = get_tickets(arg)
RESMGR.assign(res)
pprint.pprint(res)
def github_issues(arg=None):
"""
Retrieve information about cmssh tickets, e.g.
Examples:
cmssh> tickets # list all cmssh tickets
cmssh> ticket 14 # get details for given ticket id
cmssh> ticket new # post new ticket from cmssh
# or post it at https://github.com/vkuznet/cmssh/issues/new
"""
if arg == 'new':
msg = 'You can post new ticket via web interface at\n'
msg += 'https://github.com/vkuznet/cmssh/issues/new\n'
msg += 'otherwise it will be posted as anonymous gist ticket'
print_info(msg)
if not user_input('Proceed', default='N'):
return
email = raw_input('Your Email : ')
if not email:
msg = "You did your email address"
print_error(msg)
return
desc = ''
msg = 'Type your problem, attach traceback, etc. Once done print '
msg += msg_blue('EOF') + ' and hit ' + msg_blue('Enter') + '\n'
print msg
while True:
try:
uinput = raw_input()
if uinput.strip() == 'EOF':
break
desc += uinput + '\n'
except KeyboardInterrupt:
break
if not desc:
msg = "You did not provide bug description"
print_error(msg)
return
if not user_input('Send this ticket', default='N'):
print_info('Aborting your action')
return
key = 'cmssh-%s' % time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(time.time()))
files = {key: {'content': desc}}
res = post_ticket(key, files)
if res.has_key('html_url'):
print_status('New gist ticket %s' % res['html_url'])
title = 'cmssh gist %s' % res['html_url']
if isinstance(res, dict):
ticket = pprint.pformat(res)
else:
ticket = res
to_user = base64.decodestring('dmt1em5ldEBnbWFpbC5jb20=\n')
send_email(to_user, email, title, ticket)
else:
res = get_tickets(arg)
RESMGR.assign(res)
pprint.pprint(res)
def integration_tests(_arg):
"Run series of integration tests for cmssh"
for fname in ['file1.root', 'file2.root']:
if os.path.isfile(fname):
os.remove(fname)
lfn = \
'/store/data/Run2012A/ElectronHad/AOD/PromptReco-v1/000/193/686/58802521-EF9A-E111-9EE7-BCAEC518FF50.root'
lfn2 = \
'/store/data/Run2012A/ElectronHad/AOD/PromptReco-v1/000/190/450/84087548-ED80-E111-A737-0025901D5D80.root'
dataset = '/PhotonHad/Run2011A-PromptReco-v1/RECO'
dataset2 = '/SUSY_LM9_sftsht_8TeV-pythia6/Summer12-START50_V13-v1/GEN-SIM'
run = 160915
sename = 'T3_US_Cornell:/store/user/valya'
cmd_list = ['pager 0', 'debug_http 0']
cmd_list += ['ls', 'mkdir ttt', 'ls -l', 'rmdir ttt', 'ls']
cmd_list += ['ls dataset=%s' % dataset, 'ls run=%s' % run, 'ls file=%s' % lfn]
cmd_list += ['ls %s' % dataset, 'info %s' % dataset]
cmd_list += ['find dataset=/ZMM*', 'das dataset=/ZMM*', 'find dataset file=%s' % lfn]
cmd_list += ['find lumi dataset=%s' % dataset,
'find lumi {"190704":[1,2,3]}',
'find lumi {190704:[1,2,3]}']
cmd_list += ['find config dataset=%s' % dataset2]
cmd_list += ['du T3_US_Cornell', 'ls T3_US_Cornell']
cmd_list += ['ls %s' % sename,
'mkdir %s/foo' % sename,
'ls %s' % sename,
'rmdir %s/foo' % sename,
'ls %s' % sename,
]
cmd_list += ['cp %s file.root' % lfn,
'ls',
'cp file.root %s' % sename,
'ls %s' % sename,
'rm %s/file.root' % sename,
'ls %s' % sename,
'rm file.root',
'cp %s file1.root &' % lfn,
'cp %s file2.root &' % lfn2,
'ls']
cmd_list += ['find user=oliver', 'jobs list', 'jobs user=AikenOliver']
cmd_list += ['releases list', 'arch list', 'jobs', 'ls']
cmd_list += ['read https://twiki.cern.ch/twiki/bin/viewauth/CMS/SWGuideLHEtoEOS']
mgr = get_ipython()
for item in cmd_list:
print_info("Execute %s" % item)
split = item.split(' ', 1)
if len(split) == 1:
cmd = item
args = ''
else:
cmd = split[0]
args = split[-1]
mgr.run_line_magic(cmd, args)
def integration_tests(_arg):
"Run series of integration tests for cmssh"
for fname in ['file1.root', 'file2.root']:
if os.path.isfile(fname):
os.remove(fname)
lfn = \
'/store/data/CRUZET3/Cosmics/RAW/v1/000/050/832/186585EC-024D-DD11-B747-000423D94AA8.root'
lfn2 = \
'/store/data/CRUZET3/Cosmics/RAW/v1/000/050/796/4E1D3610-E64C-DD11-8629-001D09F251FE.root'
dataset = '/PhotonHad/Run2011A-PromptReco-v1/RECO'
dataset2 = '/SUSY_LM9_sftsht_8TeV-pythia6/Summer12-START50_V13-v1/GEN-SIM'
run = 160915
sename = 'T3_US_Cornell:/store/user/valya'
cmd_list = ['pager 0', 'debug_http 0']
cmd_list += ['ls', 'mkdir ttt', 'ls -l', 'rmdir ttt', 'ls']
cmd_list += ['ls dataset=%s' % dataset, 'ls run=%s' % run, 'ls file=%s' % lfn]
cmd_list += ['ls %s' % dataset, 'info %s' % dataset]
cmd_list += ['find dataset=/ZMM*', 'das dataset=/ZMM*']
cmd_list += ['find lumi dataset=%s' % dataset,
'find lumi {"190704":[1,2,3]}',
'find lumi {190704:[1,2,3]}']
cmd_list += ['find config dataset=%s' % dataset2]
cmd_list += ['du T3_US_Cornell', 'ls T3_US_Cornell']
cmd_list += ['ls %s' % sename,
'mkdir %s/foo' % sename,
'ls %s' % sename,
'rmdir %s/foo' % sename,
'ls %s' % sename,
]
cmd_list += ['cp %s file.root' % lfn,
'ls',
'cp file.root %s' % sename,
'ls %s' % sename,
'rm %s/file.root' % sename,
'ls %s' % sename,
'rm file.root',
'cp %s file1.root &' % lfn,
'cp %s file2.root &' % lfn2,
'ls']
cmd_list += ['find user=oliver', 'jobs list', 'jobs user=AikenOliver']
cmd_list += ['releases list', 'arch list', 'jobs', 'ls']
cmd_list += ['read https://twiki.cern.ch/twiki/bin/viewauth/CMS/SWGuideLHEtoEOS']
mgr = get_ipython()
for item in cmd_list:
print_info("Execute %s" % item)
split = item.split(' ', 1)
if len(split) == 1:
cmd = item
args = ''
else:
cmd = split[0]
args = split[-1]
mgr.run_line_magic(cmd, args)
def verbose(arg):
"""
Set/get verbosity level
"""
arg = arg.strip()
ipth = get_ipython()
if arg == '':
print_info("Verbose level is %s" % ipth.debug)
else:
if arg == 0 or arg == '0':
ipth.debug = False
else:
ipth.debug = True
def debug_http(arg):
"""
Show or set HTTP debug flag. Default is 0.
"""
arg = arg.strip()
if arg:
if arg not in ['0', '1']:
print_error('Please provide 0/1 for debug_http command')
return
print_info("Set HTTP debug level to %s" % arg)
os.environ['HTTPDEBUG'] = arg
else:
print_info("HTTP debug level is %s" % os.environ.get('HTTPDEBUG', 0))
def verbose(arg):
"""
Set/get verbosity level
"""
arg = arg.strip()
ipth = get_ipython()
if arg == '':
print_info("Verbose level is %s" % ipth.debug)
else:
if arg == 0 or arg == '0':
ipth.debug = False
else:
ipth.debug = True
def debug_http(arg):
"""
Show or set HTTP debug flag. Default is 0.
"""
arg = arg.strip()
if arg:
if arg not in ['0', '1']:
print_error('Please provide 0/1 for debug_http command')
return
print_info("Set HTTP debug level to %s" % arg)
os.environ['HTTPDEBUG'] = arg
else:
print_info("HTTP debug level is %s" % os.environ.get('HTTPDEBUG', 0))
def cms_json(arg):
"Print or set location of CMS JSON file"
if arg:
if access2file(arg):
os.environ['CMS_JSON'] = arg
print_info('CMS_JSON: %s' % arg)
else:
fname = os.environ.get('CMS_JSON')
print_info('CMS JSON: %s' % fname)
try:
debug = get_ipython().debug
except:
debug = 0
if debug and access2file(fname):
with open(fname, 'r') as cms_json:
print cms_json.read()
def cms_json(arg):
"Print or set location of CMS JSON file"
if arg:
if access2file(arg):
os.environ['CMS_JSON'] = arg
print_info('CMS_JSON: %s' % arg)
else:
fname = os.environ.get('CMS_JSON')
print_info('CMS JSON: %s' % fname)
try:
debug = get_ipython().debug
except:
debug = 0
if debug and access2file(fname):
with open(fname, 'r') as cms_json:
print cms_json.read()
def register(prompt, name, funcList=[]):
"""Register shell"""
set_prompt(prompt)
ID.prompt = prompt
ID.name = name
funcList.sort()
ID.dict[name] = funcList
if funcList:
print_info("Available commands within %s sub-shell:" % prompt)
if funcList:
if not funcList.count('_exit'):
funcList.append('_exit')
for func in funcList:
print_info("%s %s" % (" "*10, func))
if not ID.funcList.count(func):
ID.funcList.append(func)
else:
ID.funcList = funcList
def register(prompt, name, funcList=[]):
"""Register shell"""
set_prompt(prompt)
ID.prompt = prompt
ID.name = name
funcList.sort()
ID.dict[name] = funcList
if funcList:
print_info("Available commands within %s sub-shell:" % prompt)
if funcList:
if not funcList.count('_exit'):
funcList.append('_exit')
for func in funcList:
print_info("%s %s" % (" " * 10, func))
if not ID.funcList.count(func):
ID.funcList.append(func)
else:
ID.funcList = funcList
def mkdir(self, path, verbose=0):
"""mkdir command"""
spath = path.split(':')
if len(spath) == 1:
node = spath[0]
ldir = '/store/user'
else:
node = spath[0]
ldir = spath[1]
dst = [r for r in resolve_user_srm_path(node, ldir)][0]
cmd = '%s %s' % (os.environ.get('SRM_MKDIR', ''), dst)
if verbose:
print_info(cmd)
try:
stdout, stderr = execmd(cmd)
if verbose:
print_info(stdout + stderr)
except:
return 'fail'
return 'success'
def mkdir(self, path, verbose=0):
"""mkdir command"""
spath = path.split(':')
if len(spath) == 1:
node = spath[0]
ldir = '/store/user'
else:
node = spath[0]
ldir = spath[1]
dst = [r for r in resolve_user_srm_path(node, ldir)][0]
cmd = '%s %s' % (os.environ.get('SRM_MKDIR', ''), dst)
if verbose:
print_info(cmd)
try:
stdout, stderr = execmd(cmd)
if verbose:
print_info(stdout + stderr)
except:
return 'fail'
return 'success'
def cms_jobs(arg=None):
"""
cmssh jobs command lists local job queue or provides information
about jobs at give site or for given user. It accepts the following
list of options:
- list, which lists local transfer jobs
- site, which lists jobs at given site
- dashboard, which lists jobs of current user
- user, which lists jobs of given user
Examples:
cmssh> jobs
cmssh> jobs list
cmssh> jobs site=T2_US_UCSD
cmssh> jobs dashboard
cmssh> jobs user=my_cms_user_name
"""
res = None
try:
debug = get_ipython().debug
except:
debug = 0
orig_arg = arg
if orig_arg.find('|') != -1:
arg, flt = orig_arg.split('|', 1)
arg = arg.strip()
else:
flt = None
if arg:
arg = arg.strip()
if not arg or arg == 'list':
print_info('Local data transfer')
dqueue(arg)
elif arg == 'dashboard':
userdn = os.environ.get('USER_DN', None)
if userdn:
user = get_dashboardname(userdn)
print_info('Dashboard information, user=%s' % user)
res = jobsummary({'user': user})
elif pat_site.match(arg):
site = arg.replace('site=', '')
print_info('Dashboard information, site=%s' % site)
res = jobsummary({'site': site})
elif pat_user.match(arg):
user = arg.replace('user='******'')
print_info('Dashboard information, user=%s' % user)
res = jobsummary({'user': user})
if res:
RESMGR.assign(res)
list_results(res, debug=True, flt=flt)
def cms_jobs(arg=None):
"""
cmssh jobs command lists local job queue or provides information
about jobs at give site or for given user. It accepts the following
list of options:
- list, which lists local transfer jobs
- site, which lists jobs at given site
- dashboard, which lists jobs of current user
- user, which lists jobs of given user
Examples:
cmssh> jobs
cmssh> jobs list
cmssh> jobs site=T2_US_UCSD
cmssh> jobs dashboard
cmssh> jobs user=my_cms_user_name
"""
res = None
try:
debug = get_ipython().debug
except:
debug = 0
orig_arg = arg
if orig_arg.find('|') != -1:
arg, flt = orig_arg.split('|', 1)
arg = arg.strip()
else:
flt = None
if arg:
arg = arg.strip()
if not arg or arg == 'list':
print_info('Local data transfer')
dqueue(arg)
elif arg == 'dashboard':
userdn = os.environ.get('USER_DN', None)
if userdn:
user = get_dashboardname(userdn)
print_info('Dashboard information, user=%s' % user)
res = jobsummary({'user': user})
elif pat_site.match(arg):
site = arg.replace('site=', '')
print_info('Dashboard information, site=%s' % site)
res = jobsummary({'site': site})
elif pat_user.match(arg):
user = arg.replace('user='******'')
print_info('Dashboard information, user=%s' % user)
res = jobsummary({'user': user})
if res:
RESMGR.assign(res)
list_results(res, debug=True, flt=flt)
def reqmgr(dataset):
"Retrieve information from CMS ReqMgr data-service"
url = 'https://cmsweb.cern.ch/reqmgr/rest/configIDs' + dataset
cert = os.path.join(os.environ['HOME'], '.globus/usercert.pem')
configs = []
with working_pem(PEMMGR.pem) as key:
with get_data_and_close(url, key, cert) as data:
stream = data.read()
try:
jsondict = json.loads(stream)
except Exception as _exc:
jsondict = eval(stream, {"__builtins__": None}, {})
for key, val in jsondict.items():
for item in val:
configs.append(item)
if configs:
print_info('Found %s configuration files:' % len(configs))
for rec in configs:
print rec
else:
print_info('No configuration files is found')
def reqmgr(dataset):
"Retrieve information from CMS ReqMgr data-service"
url = 'https://cmsweb.cern.ch/reqmgr/rest/configIDs' + dataset
cert = os.path.join(os.environ['HOME'], '.globus/usercert.pem')
configs = []
with working_pem(PEMMGR.pem) as key:
with get_data_and_close(url, key, cert) as data:
stream = data.read()
try:
jsondict = json.loads(stream)
except Exception as _exc:
jsondict = eval(stream, { "__builtins__": None }, {})
for key, val in jsondict.items():
for item in val:
configs.append(item)
if configs:
print_info('Found %s configuration files:' % len(configs))
for rec in configs:
print rec
else:
print_info('No configuration files is found')
def get_size(surl, verbose=None):
"""
Execute srm-ls <surl> command and retrieve file size information
"""
srmls = os.environ.get('SRM_LS', '')
if not srmls:
print_error('Unable to find srm ls tool')
sys.exit(1)
if srmls.find('srm-ls') != -1:
srmargs = ''
else:
srmargs = '-2'
cmd = '%s %s %s' % (srmls, srmargs, surl)
if verbose:
print_info(cmd)
if cmd.find('file:///') != -1:
return file_size(cmd.split('file:///')[-1])
stdout, stderr = execmd(cmd)
if verbose:
print_info(stdout + stderr)
orig_size = 0
if cmd.find('file:///') != -1: # srm-ls returns XML
if srmls.find('srm-ls') != -1:
orig_size = parse_srmls(stdout)
else:
try:
orig_size = stdout.split()[0].strip()
except:
return 0
else:
if srmls.find('srm-ls') != -1:
for line in stdout.split('\n'):
if line.find('Bytes') != -1:
orig_size = line.replace('\n', '').split('=')[-1]
else:
try:
orig_size = stdout.split()[0].strip()
except:
return 0
return orig_size
def get_size(surl, verbose=None):
"""
Execute srm-ls <surl> command and retrieve file size information
"""
srmls = os.environ.get('SRM_LS', '')
if not srmls:
print_error('Unable to find srm ls tool')
sys.exit(1)
if srmls.find('srm-ls') != -1:
srmargs = ''
else:
srmargs = '-2'
cmd = '%s %s %s' % (srmls, srmargs, surl)
if verbose:
print_info(cmd)
if cmd.find('file:///') != -1:
return file_size(cmd.split('file:///')[-1])
stdout, stderr = execmd(cmd)
if verbose:
print_info(stdout + stderr)
orig_size = 0
if cmd.find('file:///') != -1: # srm-ls returns XML
if srmls.find('srm-ls') != -1:
orig_size = parse_srmls(stdout)
else:
try:
orig_size = stdout.split()[0].strip()
except:
return 0
else:
if srmls.find('srm-ls') != -1:
for line in stdout.split('\n'):
if line.find('Bytes') != -1:
orig_size = line.replace('\n', '').split('=')[-1]
else:
try:
orig_size = stdout.split()[0].strip()
except:
return 0
return orig_size
def cmscrab(arg):
"""
Execute CRAB command, help is available at
https://twiki.cern.ch/twiki/bin/view/CMSPublic/SWGuideCrabFaq
"""
msg = \
'CRAB FAQ: https://twiki.cern.ch/twiki/bin/view/CMSPublic/SWGuideCrabFaq'
print_info(msg)
# check if release version and work area are set (should be set at cmsrel)
rel = os.environ.get('CMSSW_VERSION', None)
work_area = os.environ.get('CMSSW_WORKAREA', None)
if not rel or not work_area:
msg = 'In order to run crab command you must '
msg += 'run ' + msg_blue('cmsrel') + ' command'
print_error(msg)
return
# check existence of crab.cfg
crab_dir = os.path.join(work_area, 'crab')
crab_cfg = os.path.join(crab_dir, 'crab.cfg')
if not os.path.isdir(crab_dir):
os.makedirs(crab_dir)
os.chdir(crab_dir)
if not os.path.isfile(crab_cfg):
msg = 'No crab.cfg file found in %s' % crab_dir
print_warning(msg)
msg = 'Would you like to create one'
if user_input(msg, default='N'):
with open('crab.cfg', 'w') as config:
config.write(crabconfig())
msg = 'Your crab.cfg has been created, please edit it '
msg += 'appropriately and re-run crab command'
print_info(msg)
print "cwd:", os.getcwd()
return
if os.uname()[0] == 'Darwin' and arg == '-submit':
crab_submit_remotely(rel, work_area)
return
cmd = 'source $CRAB_ROOT/crab.sh; crab %s' % arg
cmsexe(cmd)
def cmscrab(arg):
"""
Execute CRAB command, help is available at
https://twiki.cern.ch/twiki/bin/view/CMSPublic/SWGuideCrabFaq
"""
msg = \
'CRAB FAQ: https://twiki.cern.ch/twiki/bin/view/CMSPublic/SWGuideCrabFaq'
print_info(msg)
# check if release version and work area are set (should be set at cmsrel)
rel = os.environ.get('CMSSW_VERSION', None)
work_area = os.environ.get('CMSSW_WORKAREA', None)
if not rel or not work_area:
msg = 'In order to run crab command you must '
msg += 'run ' + msg_blue('cmsrel') + ' command'
print_error(msg)
return
# check existence of crab.cfg
crab_dir = os.path.join(work_area, 'crab')
crab_cfg = os.path.join(crab_dir, 'crab.cfg')
if not os.path.isdir(crab_dir):
os.makedirs(crab_dir)
os.chdir(crab_dir)
if not os.path.isfile(crab_cfg):
msg = 'No crab.cfg file found in %s' % crab_dir
print_warning(msg)
msg = 'Would you like to create one'
if user_input(msg, default='N'):
with open('crab.cfg', 'w') as config:
config.write(crabconfig())
msg = 'Your crab.cfg has been created, please edit it '
msg += 'appropriately and re-run crab command'
print_info(msg)
print "cwd:", os.getcwd()
return
if os.uname()[0] == 'Darwin' and arg == '-submit':
crab_submit_remotely(rel, work_area)
return
cmd = 'source $CRAB_ROOT/crab.sh; crab %s' % arg
cmsexe(cmd)
def run(cmd, cdir=None, log=None, msg=None, debug=None, shell=False, call=False):
"Run given command via subprocess call"
if msg:
print msg
cmd = cmd.strip()
kwds = {}
if shell or cmd.find(';') != -1 or cmd.find('&&') != -1 or \
cmd.find('|') != -1:
kwds = {'shell': True}
else:
if isinstance(cmd, unicode):
cmd = shlex.split(cmd.encode('ascii', 'ignore'))
else:
cmd = shlex.split(cmd)
msg = 'execute cmd=%s, kwds=%s' % (cmd, kwds)
if debug:
print_info(msg.capitalize())
try:
with working_dir(cdir):
if not call:
kwds.update({'stdout':subprocess.PIPE,
'stderr':subprocess.PIPE,
'close_fds':True})
if log:
with open(log, 'w') as logstream:
kwds.update({'stdout': logstream, 'stderr': logstream})
if call:
subprocess.call(cmd, **kwds)
return
pipe = subprocess.Popen(cmd, **kwds)
else:
if call:
subprocess.call(cmd, **kwds)
return
pipe = subprocess.Popen(cmd, **kwds)
(child_stdout, child_stderr) = (pipe.stdout, pipe.stderr)
if child_stdout:
stdout = child_stdout.read()
else:
stdout = ''
if child_stderr:
stderr = child_stderr.read()
else:
stderr = ''
if stderr:
if isinstance(cmd, list):
cmd_str = ' '.join(cmd)
else:
cmd_str = cmd
if cmd_str.find('proxy') != -1: # proxy commands prints . to stderr
sss = stderr.replace(stderr[0], '')
if not (stderr[0] == '.' and not len(sss)):
print_error(stderr)
else:
print_error(stderr)
if stdout:
print stdout
except OSError as err:
msg = 'OSError, fail to ' + msg + ', error=%s' % str(err)
print_error(msg)
except Exception as err:
msg = 'Fail to ' + msg + ', error=%s' % str(err)
print_error(msg)
def get_data_sso(url, key, cert, debug=0, redirect=None):
"""
Main routine to get data from data service behind CERN SSO.
Return file-like descriptor object (similar to open).
For iCMS access we need to pass via environment checking, e.g.
http://cms.cern.ch/iCMS/bla should be redirected to
https://cms.cern.ch/test/env.cgi?url=http://cms.cern.ch/iCMS/bla
"""
cern_env = 'https://cms.cern.ch/test/env.cgi?url='
if url.find('http://cms.cern.ch/iCMS') == 0 or\
url.find('https://cms.cern.ch/iCMS') == 0:
url = cern_env + url
orig_url = url
orig_args = url.split('?')[-1]
# send request to url, it sets the _shibstate_ cookie which
# will be used for redirection
opener = create_https_opener(key, cert)
fdesc = opener.open(url)
url = fdesc.geturl()
if url.find('?') == -1:
params = ''
else:
params = url.split('?')[-1] # redirect parameters
if int(os.environ.get('HTTPDEBUG', 0)):
print_info('Response info')
print fdesc.info()
print fdesc.geturl()
if not params or params == orig_args:
# if we did not receive new set of SSO parameters pass file
# descriptor to upper level, e.g.
# in case of CERN twiki it does not pass through SSO
# therefore we just return file descriptor
return fdesc
fdesc.close()
# now, request authentication at CERN login page
url = 'https://login.cern.ch/adfs/ls/auth/sslclient/'
if int(os.environ.get('HTTPDEBUG', 0)):
print_info('CERN Login parameters')
print url + '?' + params
if params:
fdesc = opener.open(url + '?' + params)
else:
fdesc = opener.open(url)
data = fdesc.read()
if int(os.environ.get('HTTPDEBUG', 0)):
print_info('Response info')
print fdesc.info()
fdesc.close()
# parse SSO output
param_dict, action = parse_sso_output(data)
# finally, retrieve data from given action URL/redirect or original URL
# with all cookies in place and given set of parameters
if action:
url = action
elif redirect:
url = redirect
else:
url = orig_url
params = urllib.urlencode(param_dict)
if int(os.environ.get('HTTPDEBUG', 0)):
print_info('Redirect parameters')
print url + '?' + params
fdesc = opener.open(url, params)
return fdesc
def pfn_dst(lfn, dst, verbose=None):
"""
Look-up LFN in Phedex and return pfn dst for further processing
"""
dstfname = None
pat = re.compile('^T[0-9]_[A-Z]+(_)[A-Z]+')
if pat.match(dst):
dst_split = dst.split(':')
dst = dst_split[0]
if len(dst_split) == 1: # copy to the node
local_path = dst_split[1]
for srm_path, lfn_match in resolve_srm_path(dst, verbose):
lfn_pat = re.compile(lfn_match)
if lfn_pat.match(lfn):
srm_path = srm_path.replace('\?', '?').replace('$1', local_path)
if verbose:
print "Resolve %s into %s" % (dst, srm_path)
dst = srm_path
else:
paths = [p for p in resolve_user_srm_path(dst, verbose=verbose)]
dst = '%s/%s' % (paths[0], get_username())
check_permission(dst, verbose)
else:
if dst.find('file:///') == -1:
dstfname = dst.split('/')[-1]
if dstfname == '.':
dstfname = None
if dst[0] == '/': # absolute path
if os.path.isdir(dst):
ddir = dst
dstfname = None
else:
ddir = '/'.join(dst.split('/')[:-1])
if not os.path.isdir(ddir):
msg = 'Provided destination directory %s does not exists' % ddir
raise Exception(msg)
dst = 'file:///%s' % ddir
else:
ddir = '/'.join(dst.split('/')[:-1]).replace('$PWD', os.getcwd())
if os.path.isdir(ddir):
dst = 'file:///%s' % os.path.join(os.getcwd(), ddir)
else:
dst = 'file:///%s' % os.getcwd()
pfnlist = []
if os.path.isfile(lfn) or lfn.find('file:///') != -1: # local file
pfn = lfn.replace('file:///', '')
if pfn[0] != '/':
pfn = 'file:///%s' % os.path.join(os.getcwd(), pfn)
else:
pfn = 'file:///%s' % pfn
pfnlist = [pfn]
else:
if lfn.find(':') != -1:
node, lfn = lfn.split(':')
params = {'node':node, 'lfn':lfn, 'protocol':'srmv2'}
method = 'lfn2pfn'
else:
params = {'se':'*', 'lfn':lfn}
method = 'fileReplicas'
json_dict = get_data(phedex_url(method), params)
ddict = DotDict(json_dict)
if verbose:
print "Look-up LFN:"
print lfn
phedex = json_dict['phedex']
if phedex.has_key('mapping'):
if not phedex['mapping']:
msg = "LFN: %s\n" % lfn
msg += 'No replicas found\n'
msg += str(json_dict)
raise Exception(msg)
filelist = ddict.get('phedex.mapping.pfn')
if not filelist:
filelist = []
if isinstance(filelist, basestring):
filelist = [filelist]
for fname in filelist:
pfnlist.append(fname)
elif phedex.has_key('block') and not phedex['block']:
msg = 'No replicas found in PhEDEx, will try to get original SE from DBS'
print_warning(msg)
sename = get_dbs_se(lfn)
msg = 'Orignal LFN site %s' % sename
print_info(msg)
mgr = SiteDBManager()
pfnlist = lfn2pfn(lfn, sename, mgr)
filelist = ddict.get('phedex.block.file')
if not filelist:
filelist = []
for fname in filelist:
for replica in fname['replica']:
cmsname = replica['node']
se = replica['se']
if verbose:
print "found LFN on node=%s, se=%s" % (cmsname, se)
if cmsname.count('T0', 0, 2) == 1:
continue # skip T0's
# query Phedex for PFN
params = {'protocol':'srmv2', 'lfn':lfn, 'node':cmsname}
result = get_data(phedex_url('lfn2pfn'), params)
try:
for item in result['phedex']['mapping']:
pfn = item['pfn']
if pfn not in pfnlist:
pfnlist.append(pfn)
except:
msg = "Fail to look-up PFNs in Phedex\n" + str(result)
print msg
continue
if verbose > 1:
print "PFN list:"
for pfn in pfnlist:
print pfn
# finally return pfn and dst paths w/ file for further processing
for item in pfnlist:
ifile = item.split("/")[-1] if not dstfname else dstfname
yield item, '%s/%s' % (dst, ifile)
def transfer(self, cmd, lfn, pfn, pdst, verbose=0, background=False):
"""Copy LFN to given destination"""
err = 'Unable to identify total size of the file,'
err += ' GRID middleware fails.'
if not background:
bar = PrintProgress('Fetching LFN info')
if verbose:
print_info(cmd)
if background:
# here background is a list of commands
if not isinstance(background, list):
return 'fail'
proc = Process(target=execute, args=(background, pfn, pdst, 0))
self.queue[lfn] = (proc, None)
return 'accepted'
elif verbose:
status = execute(cmd, pfn, pdst, verbose)
if not status:
return 'fail'
else:
dst, dst_size = status
size = size_format(dst_size)
if not size or not dst_size:
print_error(err)
print "Status of transfer:\n", status
return 'fail'
else:
print "\nDone, file located at %s (%s)" \
% (dst, size_format(dst_size))
return 'success'
else:
ifile = pdst
pfn_size = get_size(pfn)
if pfn_size and pfn_size != 'null':
tot_size = float(pfn_size)
bar.print_msg('LFN size=%s' % size_format(tot_size))
bar.init('Download in progress:')
proc = Process(target=execute, args=(cmd, pfn, pdst, verbose))
proc.start()
while True:
if proc.is_alive():
size = get_size(ifile)
if not size or size == 'null':
bar.refresh('')
pass
else:
progress = float(size) * 100 / tot_size
bar.refresh(progress)
if progress == 100:
break
else:
break
time.sleep(0.5)
bar.clear()
status = check_file(pfn, pdst, verbose)
if status:
return 'success'
else:
print_error(err)
return 'fail'
return 'fail'
def pfn_dst(lfn, dst, verbose=None):
"""
Look-up LFN in Phedex and return pfn dst for further processing
"""
dstfname = None
pat = re.compile('^T[0-9]_[A-Z]+(_)[A-Z]+')
if pat.match(dst):
dst_split = dst.split(':')
dst = dst_split[0]
if len(dst_split) == 1: # copy to the node
local_path = dst_split[1]
for srm_path, lfn_match in resolve_srm_path(dst, verbose):
lfn_pat = re.compile(lfn_match)
if lfn_pat.match(lfn):
srm_path = srm_path.replace('\?',
'?').replace('$1', local_path)
if verbose:
print "Resolve %s into %s" % (dst, srm_path)
dst = srm_path
else:
paths = [p for p in resolve_user_srm_path(dst, verbose=verbose)]
dst = '%s/%s' % (paths[0], get_username())
check_permission(dst, verbose)
else:
if dst.find('file:///') == -1:
dstfname = dst.split('/')[-1]
if dstfname == '.':
dstfname = None
if dst[0] == '/': # absolute path
if os.path.isdir(dst):
ddir = dst
dstfname = None
else:
ddir = '/'.join(dst.split('/')[:-1])
if not os.path.isdir(ddir):
msg = 'Provided destination directory %s does not exists' % ddir
raise Exception(msg)
dst = 'file:///%s' % ddir
else:
ddir = '/'.join(dst.split('/')[:-1]).replace(
'$PWD', os.getcwd())
if os.path.isdir(ddir):
dst = 'file:///%s' % os.path.join(os.getcwd(), ddir)
else:
dst = 'file:///%s' % os.getcwd()
pfnlist = []
if os.path.isfile(lfn) or lfn.find('file:///') != -1: # local file
pfn = lfn.replace('file:///', '')
if pfn[0] != '/':
pfn = 'file:///%s' % os.path.join(os.getcwd(), pfn)
else:
pfn = 'file:///%s' % pfn
pfnlist = [pfn]
else:
if lfn.find(':') != -1:
node, lfn = lfn.split(':')
params = {'node': node, 'lfn': lfn, 'protocol': 'srmv2'}
method = 'lfn2pfn'
else:
params = {'se': '*', 'lfn': lfn}
method = 'fileReplicas'
json_dict = get_data(phedex_url(method), params)
ddict = DotDict(json_dict)
if verbose:
print "Look-up LFN:"
print lfn
phedex = json_dict['phedex']
if phedex.has_key('mapping'):
if not phedex['mapping']:
msg = "LFN: %s\n" % lfn
msg += 'No replicas found\n'
msg += str(json_dict)
raise Exception(msg)
filelist = ddict.get('phedex.mapping.pfn')
if not filelist:
filelist = []
if isinstance(filelist, basestring):
filelist = [filelist]
for fname in filelist:
pfnlist.append(fname)
elif phedex.has_key('block') and not phedex['block']:
msg = 'No replicas found in PhEDEx, will try to get original SE from DBS'
print_warning(msg)
sename = get_dbs_se(lfn)
msg = 'Orignal LFN site %s' % sename
print_info(msg)
mgr = SiteDBManager()
pfnlist = lfn2pfn(lfn, sename, mgr)
filelist = ddict.get('phedex.block.file')
if not filelist:
filelist = []
for fname in filelist:
for replica in fname['replica']:
cmsname = replica['node']
se = replica['se']
if verbose:
print "found LFN on node=%s, se=%s" % (cmsname, se)
if cmsname.count('T0', 0, 2) == 1:
continue # skip T0's
# query Phedex for PFN
params = {'protocol': 'srmv2', 'lfn': lfn, 'node': cmsname}
result = get_data(phedex_url('lfn2pfn'), params)
try:
for item in result['phedex']['mapping']:
pfn = item['pfn']
if pfn not in pfnlist:
pfnlist.append(pfn)
except:
msg = "Fail to look-up PFNs in Phedex\n" + str(result)
print msg
continue
if verbose > 1:
print "PFN list:"
for pfn in pfnlist:
print pfn
# finally return pfn and dst paths w/ file for further processing
for item in pfnlist:
ifile = item.split("/")[-1] if not dstfname else dstfname
yield item, '%s/%s' % (dst, ifile)
def read(url, output=None, debug=0):
"Get run information from RunSummary data-service"
encoding = 'utf-8'
key = None
cert = os.path.join(os.environ['HOME'], '.globus/usercert.pem')
if os.path.isfile(url):
with open(url, 'r') as stream:
context = stream.read()
try:
pydoc.pager(context)
except:
print context
return
elif url.find('cmsweb.cern.ch') != -1:
data = get_data(url, decoder=None)
html = data
encoding = None
elif url.find('mcdb.cern.ch') != -1:
data = urllib.urlopen(url)
html = data.read().replace(' _place_holder;', '')
encoding = enc(data.headers, html)[0]
elif url.find('cern.ch') == -1:
data = urllib.urlopen(url)
html = data.read()
encoding = enc(data.headers, html)[0]
else:
with working_pem(PEMMGR.pem) as key:
data = get_data_sso(url, key, cert, debug)
html = data.read()
encoding = enc(data.headers, html)[0]
if encoding == 'us-ascii':
encoding = 'utf-8'
pager = os.environ.get('CMSSH_PAGER', None)
if html:
if int(os.environ.get('HTTPDEBUG', 0)):
print_info('read data')
print html
if encoding:
text = html.decode(encoding)
res = html2text(text, '')
if output:
with open(output, 'w') as stream:
stream.write(html)
else:
try:
if pager:
pydoc.pager(res.encode('utf-8'))
else:
wrapwrite(html2text(text, ''))
except:
wrapwrite(html2text(text, ''))
else:
if output:
with open(output, 'w') as stream:
stream.write(html)
else:
try:
if pager:
pydoc.pager(html)
else:
print html
except:
print html
def read(url, output=None, debug=0):
"Get run information from RunSummary data-service"
encoding = 'utf-8'
key = None
cert = os.path.join(os.environ['HOME'], '.globus/usercert.pem')
if os.path.isfile(url):
with open(url, 'r') as stream:
context = stream.read()
try:
pydoc.pager(context)
except:
print context
return
elif url.find('cmsweb.cern.ch') != -1:
data = get_data(url, decoder=None)
html = data
encoding = None
elif url.find('mcdb.cern.ch') != -1:
data = urllib.urlopen(url)
html = data.read().replace(' _place_holder;', '')
encoding = enc(data.headers, html)[0]
elif url.find('cern.ch') == -1:
data = urllib.urlopen(url)
html = data.read()
encoding = enc(data.headers, html)[0]
else:
with working_pem(PEMMGR.pem) as key:
data = get_data_sso(url, key, cert, debug)
html = data.read()
encoding = enc(data.headers, html)[0]
if encoding == 'us-ascii':
encoding = 'utf-8'
pager = os.environ.get('CMSSH_PAGER', None)
if html:
if int(os.environ.get('HTTPDEBUG', 0)):
print_info('read data')
print html
if encoding:
text = html.decode(encoding)
res = html2text(text, '')
if output:
with open(output, 'w') as stream:
stream.write(html)
else:
try:
if pager:
pydoc.pager(res.encode('utf-8'))
else:
wrapwrite(html2text(text, ''))
except:
wrapwrite(html2text(text, ''))
else:
if output:
with open(output, 'w') as stream:
stream.write(html)
else:
try:
if pager:
pydoc.pager(html)
else:
print html
except:
print html
def transfer(self, cmd, lfn, pfn, pdst, verbose=0, background=False):
"""Copy LFN to given destination"""
err = 'Unable to identify total size of the file,'
err += ' GRID middleware fails.'
if not background:
bar = PrintProgress('Fetching LFN info')
if verbose:
print_info(cmd)
if background:
# here background is a list of commands
if not isinstance(background, list):
return 'fail'
proc = Process(target=execute, args=(background, pfn, pdst, 0))
self.queue[lfn] = (proc, None)
return 'accepted'
elif verbose:
status = execute(cmd, pfn, pdst, verbose)
if not status:
return 'fail'
else:
dst, dst_size = status
size = size_format(dst_size)
if not size or not dst_size:
print_error(err)
print "Status of transfer:\n", status
return 'fail'
else:
print "\nDone, file located at %s (%s)" \
% (dst, size_format(dst_size))
return 'success'
else:
ifile = pdst
pfn_size = get_size(pfn)
if pfn_size and pfn_size != 'null':
tot_size = float(pfn_size)
bar.print_msg('LFN size=%s' % size_format(tot_size))
bar.init('Download in progress:')
proc = Process(target=execute, args=(cmd, pfn, pdst, verbose))
proc.start()
while True:
if proc.is_alive():
size = get_size(ifile)
if not size or size == 'null':
bar.refresh('')
pass
else:
progress = float(size)*100/tot_size
bar.refresh(progress)
if progress == 100:
break
else:
break
time.sleep(0.5)
bar.clear()
status = check_file(pfn, pdst, verbose)
if status:
return 'success'
else:
print_error(err)
return 'fail'
return 'fail'
def run(cmd,
cdir=None,
log=None,
msg=None,
debug=None,
shell=False,
call=False):
"Run given command via subprocess call"
if msg:
print msg
cmd = cmd.strip()
kwds = {}
if shell or cmd.find(';') != -1 or cmd.find('&&') != -1 or \
cmd.find('|') != -1:
kwds = {'shell': True}
else:
if isinstance(cmd, unicode):
cmd = shlex.split(cmd.encode('ascii', 'ignore'))
else:
cmd = shlex.split(cmd)
msg = 'execute cmd=%s, kwds=%s' % (cmd, kwds)
if debug:
print_info(msg.capitalize())
try:
with working_dir(cdir):
if not call:
kwds.update({
'stdout': subprocess.PIPE,
'stderr': subprocess.PIPE,
'close_fds': True
})
if log:
with open(log, 'w') as logstream:
kwds.update({'stdout': logstream, 'stderr': logstream})
if call:
subprocess.call(cmd, **kwds)
return
pipe = subprocess.Popen(cmd, **kwds)
else:
if call:
subprocess.call(cmd, **kwds)
return
pipe = subprocess.Popen(cmd, **kwds)
(child_stdout, child_stderr) = (pipe.stdout, pipe.stderr)
if child_stdout:
stdout = child_stdout.read()
else:
stdout = ''
if child_stderr:
stderr = child_stderr.read()
else:
stderr = ''
if stderr:
if isinstance(cmd, list):
cmd_str = ' '.join(cmd)
else:
cmd_str = cmd
if cmd_str.find(
'proxy') != -1: # proxy commands prints . to stderr
sss = stderr.replace(stderr[0], '')
if not (stderr[0] == '.' and not len(sss)):
print_error(stderr)
else:
print_error(stderr)
if stdout:
print stdout
except OSError as err:
msg = 'OSError, fail to ' + msg + ', error=%s' % str(err)
print_error(msg)
except Exception as err:
msg = 'Fail to ' + msg + ', error=%s' % str(err)
print_error(msg)