def execute(cmds, src, dst, verbose):
"""
Execute given command, but also check if file is in place at dst
"""
status = check_file(src, dst, verbose)
if status:
return status
else:
if isinstance(cmds, basestring):
stdout, stderr = execmd(cmds)
if verbose:
print_info('Output of %s' % cmd)
print stdout + stderr
status = check_file(src, dst, verbose)
elif isinstance(cmds, list):
for cmd in cmds:
if not cmd:
continue
stdout, stderr = execmd(cmd)
if verbose:
print_info('Output of %s' % cmd)
print stdout + stderr
status = check_file(src, dst, verbose)
if status:
return status
return status
def execute(cmds, src, dst, verbose):
"""
Execute given command, but also check if file is in place at dst
"""
status = check_file(src, dst, verbose)
if status:
return status
else:
if isinstance(cmds, basestring):
stdout, stderr = execmd(cmds)
if verbose:
print_info('Output of %s' % cmd)
print stdout + stderr
status = check_file(src, dst, verbose)
elif isinstance(cmds, list):
for cmd in cmds:
if not cmd:
continue
stdout, stderr = execmd(cmd)
if verbose:
print_info('Output of %s' % cmd)
print stdout + stderr
status = check_file(src, dst, verbose)
if status:
return status
return status
def rm_lfn(self, arg, verbose=0):
"""Remove user lfn from a node"""
try:
node, lfn = arg.split(':')
except:
msg = 'Given argument "%s" does not represent SE:LFN' % arg
raise Exception(msg)
cmd = os.environ.get('SRM_RM', '')
dst = [r for r in resolve_user_srm_path(node)][0]
dst, path = dst.split('=')
if dst[-1] != '=':
dst += '='
for item in lfn.split('/'):
if not item or item in path:
continue
path += '/%s' % item
cmd = "%s %s" % (cmd, dst+path)
if verbose:
print cmd
try:
stdout, stderr = execmd(cmd)
if verbose:
print_info(stdout + stderr)
except:
return 'fail'
return 'success'
def rm_lfn(self, arg, verbose=0):
"""Remove user lfn from a node"""
try:
node, lfn = arg.split(':')
except:
msg = 'Given argument "%s" does not represent SE:LFN' % arg
raise Exception(msg)
cmd = os.environ.get('SRM_RM', '')
dst = [r for r in resolve_user_srm_path(node)][0]
dst, path = dst.split('=')
if dst[-1] != '=':
dst += '='
for item in lfn.split('/'):
if not item or item in path:
continue
path += '/%s' % item
cmd = "%s %s" % (cmd, dst + path)
if verbose:
print cmd
try:
stdout, stderr = execmd(cmd)
if verbose:
print_info(stdout + stderr)
except:
return 'fail'
return 'success'
def check_software(softlist):
"""
Perform the check that Grid middleware is installed on a node
"""
help = 'Please run with --help for more options'
for cmd in softlist:
stdout, stderr = execmd(cmd)
if not stdout:
print 'Unable to find %s' % cmd
print help
sys.exit(1)
def check_software(softlist):
"""
Perform the check that Grid middleware is installed on a node
"""
help = 'Please run with --help for more options'
for cmd in softlist:
stdout, stderr = execmd(cmd)
if not stdout:
print 'Unable to find %s' % cmd
print help
sys.exit(1)
def cms_vomsinit(_arg=None):
"""
cmssh command which executes voms-proxy-init on behalf of the user
Examples:
cmssh> vomsinit
By default it applies the following options
-rfc -voms cms:/cms -key <userkey.pem> -cert <usercert.pem>
"""
cert = os.path.join(os.environ['HOME'], '.globus/usercert.pem')
with working_pem(PEMMGR.pem) as key:
run("voms-proxy-destroy")
cmd = "voms-proxy-init -rfc -voms cms:/cms -key %s -cert %s" % (key, cert)
run(cmd)
userdn = os.environ.get('USER_DN', '')
if not userdn:
cmd = "voms-proxy-info -identity"
stdout, stderr = execmd(cmd)
os.environ['USER_DN'] = stdout.replace('\n', '')
def cms_vomsinit(_arg=None):
"""
cmssh command which executes voms-proxy-init on behalf of the user
Examples:
cmssh> vomsinit
By default it applies the following options
-rfc -voms cms:/cms -key <userkey.pem> -cert <usercert.pem>
"""
cert = os.path.join(os.environ['HOME'], '.globus/usercert.pem')
with working_pem(PEMMGR.pem) as key:
run("voms-proxy-destroy")
cmd = "voms-proxy-init -rfc -voms cms:/cms -key %s -cert %s" % (key,
cert)
run(cmd)
userdn = os.environ.get('USER_DN', '')
if not userdn:
cmd = "voms-proxy-info -identity"
stdout, stderr = execmd(cmd)
os.environ['USER_DN'] = stdout.replace('\n', '')
def mkdir(self, path, verbose=0):
"""mkdir command"""
spath = path.split(':')
if len(spath) == 1:
node = spath[0]
ldir = '/store/user'
else:
node = spath[0]
ldir = spath[1]
dst = [r for r in resolve_user_srm_path(node, ldir)][0]
cmd = '%s %s' % (os.environ.get('SRM_MKDIR', ''), dst)
if verbose:
print_info(cmd)
try:
stdout, stderr = execmd(cmd)
if verbose:
print_info(stdout + stderr)
except:
return 'fail'
return 'success'
def mkdir(self, path, verbose=0):
"""mkdir command"""
spath = path.split(':')
if len(spath) == 1:
node = spath[0]
ldir = '/store/user'
else:
node = spath[0]
ldir = spath[1]
dst = [r for r in resolve_user_srm_path(node, ldir)][0]
cmd = '%s %s' % (os.environ.get('SRM_MKDIR', ''), dst)
if verbose:
print_info(cmd)
try:
stdout, stderr = execmd(cmd)
if verbose:
print_info(stdout + stderr)
except:
return 'fail'
return 'success'
def list_se(self, arg, verbose=0):
"""list content of given directory on SE"""
try:
node, ldir = arg.split(':')
except:
msg = 'Given argument "%s" does not represent SE:dir' % arg
raise Exception(msg)
srmls = os.environ.get('SRM_LS', '')
if not srmls:
print_error('Unable to find srm ls tool')
sys.exit(1)
dst = [r for r in resolve_user_srm_path(node, ldir)][0]
if os.environ.get('LCG_LS', ''):
cmd = "%s -l -v -b -D srmv2 %s" % (os.environ['LCG_LS'], dst)
else:
if srmls.find('srm-ls') != -1:
cmd = "%s %s -fulldetailed" % (srmls, dst)
else:
cmd = "%s -2 -l %s" % (srmls, dst)
if verbose:
print cmd
stdout, stderr = execmd(cmd)
if stderr:
print_error(stderr)
output = []
row = {}
if os.environ.get('LCG_LS', ''):
for line in stdout.split('\n'):
if line.find('SE type') != -1:
continue
output.append(line)
return '\n'.join(output)
elif srmls.find('srmls') != -1:
for line in srmls_printer(stdout, dst.split('=')[-1]):
output.append(line)
return '\n'.join(output)
else:
for line in srm_ls_printer(stdout, dst.split('=')[-1]):
output.append(line)
return '\n'.join(output)
def list_se(self, arg, verbose=0):
"""list content of given directory on SE"""
try:
node, ldir = arg.split(':')
except:
msg = 'Given argument "%s" does not represent SE:dir' % arg
raise Exception(msg)
srmls = os.environ.get('SRM_LS', '')
if not srmls:
print_error('Unable to find srm ls tool')
sys.exit(1)
dst = [r for r in resolve_user_srm_path(node, ldir)][0]
if os.environ.get('LCG_LS', ''):
cmd = "%s -l -v -b -D srmv2 %s" % (os.environ['LCG_LS'], dst)
else:
if srmls.find('srm-ls') != -1:
cmd = "%s %s -fulldetailed" % (srmls, dst)
else:
cmd = "%s -2 -l %s" % (srmls, dst)
if verbose:
print cmd
stdout, stderr = execmd(cmd)
if stderr:
print_error(stderr)
output = []
row = {}
if os.environ.get('LCG_LS', ''):
for line in stdout.split('\n'):
if line.find('SE type') != -1:
continue
output.append(line)
return '\n'.join(output)
elif srmls.find('srmls') != -1:
for line in srmls_printer(stdout, dst.split('=')[-1]):
output.append(line)
return '\n'.join(output)
else:
for line in srm_ls_printer(stdout, dst.split('=')[-1]):
output.append(line)
return '\n'.join(output)
def get_size(surl, verbose=None):
"""
Execute srm-ls <surl> command and retrieve file size information
"""
srmls = os.environ.get('SRM_LS', '')
if not srmls:
print_error('Unable to find srm ls tool')
sys.exit(1)
if srmls.find('srm-ls') != -1:
srmargs = ''
else:
srmargs = '-2'
cmd = '%s %s %s' % (srmls, srmargs, surl)
if verbose:
print_info(cmd)
if cmd.find('file:///') != -1:
return file_size(cmd.split('file:///')[-1])
stdout, stderr = execmd(cmd)
if verbose:
print_info(stdout + stderr)
orig_size = 0
if cmd.find('file:///') != -1: # srm-ls returns XML
if srmls.find('srm-ls') != -1:
orig_size = parse_srmls(stdout)
else:
try:
orig_size = stdout.split()[0].strip()
except:
return 0
else:
if srmls.find('srm-ls') != -1:
for line in stdout.split('\n'):
if line.find('Bytes') != -1:
orig_size = line.replace('\n', '').split('=')[-1]
else:
try:
orig_size = stdout.split()[0].strip()
except:
return 0
return orig_size
def get_size(surl, verbose=None):
"""
Execute srm-ls <surl> command and retrieve file size information
"""
srmls = os.environ.get('SRM_LS', '')
if not srmls:
print_error('Unable to find srm ls tool')
sys.exit(1)
if srmls.find('srm-ls') != -1:
srmargs = ''
else:
srmargs = '-2'
cmd = '%s %s %s' % (srmls, srmargs, surl)
if verbose:
print_info(cmd)
if cmd.find('file:///') != -1:
return file_size(cmd.split('file:///')[-1])
stdout, stderr = execmd(cmd)
if verbose:
print_info(stdout + stderr)
orig_size = 0
if cmd.find('file:///') != -1: # srm-ls returns XML
if srmls.find('srm-ls') != -1:
orig_size = parse_srmls(stdout)
else:
try:
orig_size = stdout.split()[0].strip()
except:
return 0
else:
if srmls.find('srm-ls') != -1:
for line in stdout.split('\n'):
if line.find('Bytes') != -1:
orig_size = line.replace('\n', '').split('=')[-1]
else:
try:
orig_size = stdout.split()[0].strip()
except:
return 0
return orig_size
def get_username(verbose=None):
"""
Get user name from provided DN
"""
# get DN from grid-proxy-info
cmd = 'grid-proxy-info'
stdout, stderr = execmd(cmd)
if stderr.find('command not found') != -1:
raise Exception(stderr)
userdn = None
try:
for line in stdout.split('\n'):
if line.find('issuer') != -1:
issuer, userdn = line.split(' : ')
except:
raise Exception('Unable to parse grid-proxy-info:\n%s' % stdout)
if verbose:
print "userdn :", userdn
if not userdn:
msg = 'Unable to determine your DN, please run grid-proxy-init'
raise Exception(msg)
mgr = SiteDBManager()
user = mgr.get_user(userdn)
return user
def check_permission(dst, verbose=None):
"""
Check permission to write to given destination area
"""
if verbose:
print "Check permission to write to %s" % dst
srmmkdir = os.environ.get('SRM_MKDIR', '')
if not srmmkdir:
print_error('Unable to find srm mkdir command')
sys.exit(1)
cmd = '%s %s' % (srmmkdir, dst)
stdout, stderr = execmd(cmd)
if stderr.find('command not found') != -1:
print 'Unable to find srm mkdir tool'
print help
sys.exit(1)
if stdout.find('SRM-DIR: directory not created') != -1 or\
stdout.find('SRM_FAILURE') != -1:
msg = "Unable to access %s:" % dst
print msg
print "-" * len(msg)
print
print stdout
sys.exit(1)
def check_permission(dst, verbose=None):
"""
Check permission to write to given destination area
"""
if verbose:
print "Check permission to write to %s" % dst
srmmkdir = os.environ.get('SRM_MKDIR', '')
if not srmmkdir:
print_error('Unable to find srm mkdir command')
sys.exit(1)
cmd = '%s %s' % (srmmkdir, dst)
stdout, stderr = execmd(cmd)
if stderr.find('command not found') != -1:
print 'Unable to find srm mkdir tool'
print help
sys.exit(1)
if stdout.find('SRM-DIR: directory not created') != -1 or\
stdout.find('SRM_FAILURE') != -1:
msg = "Unable to access %s:" % dst
print msg
print "-" * len(msg)
print
print stdout
sys.exit(1)
def get_username(verbose=None):
"""
Get user name from provided DN
"""
# get DN from grid-proxy-info
cmd = 'grid-proxy-info'
stdout, stderr = execmd(cmd)
if stderr.find('command not found') != -1:
raise Exception(stderr)
userdn = None
try:
for line in stdout.split('\n'):
if line.find('issuer') != -1:
issuer, userdn = line.split(' : ')
except:
raise Exception('Unable to parse grid-proxy-info:\n%s' % stdout)
if verbose:
print "userdn :", userdn
if not userdn:
msg = 'Unable to determine your DN, please run grid-proxy-init'
raise Exception(msg)
mgr = SiteDBManager()
user = mgr.get_user(userdn)
return user
def cmsrel(rel):
"""
cmssh release setup command, it setups CMSSW environment and creates user based
directory structure.
Examples:
cmssh> cmsrel # reset CMSSW environment to cmssh one
cmssh> cmsrel CMSSW_5_2_4
"""
ipython = get_ipython()
rel = rel.strip()
if not rel or rel in ['reset', 'clear', 'clean']:
path = os.environ['CMSSH_ROOT']
for idir in ['external', 'lib', 'root']:
pdir = os.path.join(path, 'install/lib/release_%s' % idir)
if os.path.islink(pdir):
os.remove(pdir)
if os.path.isdir(pdir):
shutil.rmtree(pdir)
os.makedirs(pdir)
# Set cmssh prompt
prompt = 'cms-sh'
ipython.prompt_manager.in_template = '%s|\#> ' % prompt
return
# check if given release name is installed on user system
rel_arch = None
for arch in cms_architectures():
rel_dir = '%s/cms/cmssw/%s' % (arch, rel)
if os.path.isdir(os.path.join(os.environ['VO_CMS_SW_DIR'], rel_dir)):
rel_arch = arch
break
if not rel_arch:
msg = 'Release ' + msg_red(rel)
msg += ' is not yet installed on your system.\n'
msg += 'Use ' + msg_green('releases')
msg += ' command to list available releases.\n'
msg += 'Use ' + msg_green('install %s' % rel)
msg += ' command to install given release.'
print msg
return
# set release architecture
os.environ['SCRAM_ARCH'] = rel_arch
# setup environment
cmssw_dir = os.environ.get('CMSSW_RELEASES', os.getcwd())
if not os.path.isdir(cmssw_dir):
os.makedirs(cmssw_dir)
root = os.environ['CMSSH_ROOT']
idir = os.environ['CMSSH_INSTALL_DIR']
base = os.path.realpath('%s/CMSSW' % root)
path = '%s/%s/cms/cmssw/%s' % (base, rel_arch, rel)
os.environ['CMSSW_BASE'] = os.path.join(cmssw_dir, rel)
os.environ['CMSSW_RELEASE_BASE'] = path
for pkg in ['FWCore', 'DataFormats']:
pdir = '%s/%s' % (idir, pkg)
if os.path.exists(pdir):
shutil.rmtree(pdir)
os.mkdir(pdir)
touch(os.path.join(pdir, '__init__.py'))
pkgs = [
'Framework', 'GuiBrowsers', 'Integration', 'MessageLogger',
'MessageService', 'Modules', 'ParameterSet', 'PythonUtilities',
'Services', 'Utilities'
]
for pkg in pkgs:
link = '%s/src/FWCore/%s/python' % (path, pkg)
dst = '%s/FWCore/%s' % (idir, pkg)
os.symlink(link, dst)
link = '%s/src/DataFormats/FWLite/python' % path
dst = '%s/DataFormats/FWLite' % idir
os.symlink(link, dst)
for lib in ['external', 'lib']:
link = '%s/%s/%s' % (path, lib, rel_arch)
dst = '%s/install/lib/release_%s' % (root, lib)
if os.path.islink(dst):
os.remove(dst)
else:
shutil.rmtree(dst)
os.symlink(link, dst)
# switch to given release
os.environ['CMSSW_VERSION'] = rel
os.environ['CMSSW_WORKAREA'] = os.path.join(cmssw_dir, rel)
if os.path.isdir(os.path.join(cmssw_dir, rel + '/src')):
os.chdir(os.path.join(cmssw_dir, rel + '/src'))
else:
os.chdir(cmssw_dir)
cmd = "scramv1 project CMSSW %s" % rel
run(cmd)
os.chdir(os.path.join(rel, 'src'))
# get ROOT from run-time environment
cmd = 'eval `scramv1 runtime -sh`; env | grep ^ROOTSYS='
stdout, _stderr = execmd(cmd)
rootsys = stdout.replace('\n', '').replace('ROOTSYS=', '')
dst = '%s/install/lib/release_root' % root
if os.path.exists(dst):
if os.path.islink(dst):
os.remove(dst)
else:
shutil.rmtree(dst)
os.symlink(rootsys, dst)
# set edm utils for given release
ipython = get_ipython()
rdir = '%s/bin/%s' % (rel_dir, rel_arch)
reldir = os.path.join(os.environ['VO_CMS_SW_DIR'], rdir)
for name in os.listdir(reldir):
fname = os.path.join(reldir, name)
if name.find('edm') == 0 and os.path.isfile(fname):
# we use Magic(cmd).execute we don't need
# to add scramv1 command in front of edm one, since
# execute method will run in current shell environment
# old command for reference:
# cmd = "eval `scramv1 runtime -sh`; %s" % fname
cmd = fname
ipython.register_magic_function(Magic(cmd).execute, 'line', name)
# Set cmssh prompt
ipython.prompt_manager.in_template = '%s|\#> ' % rel
# final message
print "%s is ready, cwd: %s" % (rel, os.getcwd())
def cmsrel(rel):
"""
cmssh release setup command, it setups CMSSW environment and creates user based
directory structure.
Examples:
cmssh> cmsrel # reset CMSSW environment to cmssh one
cmssh> cmsrel CMSSW_5_2_4
"""
ipython = get_ipython()
rel = rel.strip()
if not rel or rel in ['reset', 'clear', 'clean']:
path = os.environ['CMSSH_ROOT']
for idir in ['external', 'lib', 'root']:
pdir = os.path.join(path, 'install/lib/release_%s' % idir)
if os.path.islink(pdir):
os.remove(pdir)
if os.path.isdir(pdir):
shutil.rmtree(pdir)
os.makedirs(pdir)
# Set cmssh prompt
prompt = 'cms-sh'
ipython.prompt_manager.in_template = '%s|\#> ' % prompt
return
# check if given release name is installed on user system
rel_arch = None
for arch in cms_architectures():
rel_dir = '%s/cms/cmssw/%s' % (arch, rel)
if os.path.isdir(os.path.join(os.environ['VO_CMS_SW_DIR'], rel_dir)):
rel_arch = arch
break
if not rel_arch:
msg = 'Release ' + msg_red(rel)
msg += ' is not yet installed on your system.\n'
msg += 'Use ' + msg_green('releases')
msg += ' command to list available releases.\n'
msg += 'Use ' + msg_green('install %s' % rel)
msg += ' command to install given release.'
print msg
return
# set release architecture
os.environ['SCRAM_ARCH'] = rel_arch
# setup environment
cmssw_dir = os.environ.get('CMSSW_RELEASES', os.getcwd())
if not os.path.isdir(cmssw_dir):
os.makedirs(cmssw_dir)
root = os.environ['CMSSH_ROOT']
idir = os.environ['CMSSH_INSTALL_DIR']
base = os.path.realpath('%s/CMSSW' % root)
path = '%s/%s/cms/cmssw/%s' % (base, rel_arch, rel)
os.environ['CMSSW_BASE'] = os.path.join(cmssw_dir, rel)
os.environ['CMSSW_RELEASE_BASE'] = path
for pkg in ['FWCore', 'DataFormats']:
pdir = '%s/%s' % (idir, pkg)
if os.path.exists(pdir):
shutil.rmtree(pdir)
os.mkdir(pdir)
touch(os.path.join(pdir, '__init__.py'))
pkgs = ['Framework', 'GuiBrowsers', 'Integration', 'MessageLogger',
'MessageService', 'Modules', 'ParameterSet', 'PythonUtilities',
'Services', 'Utilities']
for pkg in pkgs:
link = '%s/src/FWCore/%s/python' % (path, pkg)
dst = '%s/FWCore/%s' % (idir, pkg)
os.symlink(link, dst)
link = '%s/src/DataFormats/FWLite/python' % path
dst = '%s/DataFormats/FWLite' % idir
os.symlink(link, dst)
for lib in ['external', 'lib']:
link = '%s/%s/%s' % (path, lib, rel_arch)
dst = '%s/install/lib/release_%s' % (root, lib)
if os.path.islink(dst):
os.remove(dst)
else:
shutil.rmtree(dst)
os.symlink(link, dst)
# switch to given release
os.environ['CMSSW_VERSION'] = rel
os.environ['CMSSW_WORKAREA'] = os.path.join(cmssw_dir, rel)
if os.path.isdir(os.path.join(cmssw_dir, rel + '/src')):
os.chdir(os.path.join(cmssw_dir, rel + '/src'))
else:
os.chdir(cmssw_dir)
cmd = "scramv1 project CMSSW %s" % rel
run(cmd)
os.chdir(os.path.join(rel, 'src'))
# get ROOT from run-time environment
cmd = 'eval `scramv1 runtime -sh`; env | grep ^ROOTSYS='
stdout, _stderr = execmd(cmd)
rootsys = stdout.replace('\n', '').replace('ROOTSYS=', '')
dst = '%s/install/lib/release_root' % root
if os.path.exists(dst):
if os.path.islink(dst):
os.remove(dst)
else:
shutil.rmtree(dst)
os.symlink(rootsys, dst)
# set edm utils for given release
ipython = get_ipython()
rdir = '%s/bin/%s' % (rel_dir, rel_arch)
reldir = os.path.join(os.environ['VO_CMS_SW_DIR'], rdir)
for name in os.listdir(reldir):
fname = os.path.join(reldir, name)
if name.find('edm') == 0 and os.path.isfile(fname):
# we use Magic(cmd).execute we don't need
# to add scramv1 command in front of edm one, since
# execute method will run in current shell environment
# old command for reference:
# cmd = "eval `scramv1 runtime -sh`; %s" % fname
cmd = fname
ipython.register_magic_function(Magic(cmd).execute, 'line', name)
# Set cmssh prompt
ipython.prompt_manager.in_template = '%s|\#> ' % rel
# final message
print "%s is ready, cwd: %s" % (rel, os.getcwd())