def check(remotepath, *args, **kwargs):
"""Check if everything is alright with the files."""
verbose = kwargs.pop('verbose', False)
quiet = kwargs.pop('quiet', False)
ses = kwargs.pop('se', [])
checksum = kwargs.pop('checksum', False)
if checksum == False and len(ses) == 0:
raise InteractiveException("No check specified.")
if t2kdm.is_dir(remotepath):
raise InteractiveException(
"%s is a directory. Maybe you want to use the `--recursive` option?"
% (remotepath, ))
if verbose and len(ses) > 0:
print_("Checking replicas...")
ret = t2kdm.check_replicas(remotepath, ses, cached=True)
if not ret and not quiet:
print_("%s is not replicated on all SEs!" % (remotepath))
if checksum:
if verbose:
print_("Checking checksums...")
chk = t2kdm.check_checksums(remotepath, cached=True)
if not chk and not quiet:
print_("%s has faulty checksums!" % (remotepath))
ret = ret and chk
if ret == True:
return 0
else:
return 1
def completedefault(self, text, line, begidx, endidx):
"""Complete with content of current remote or local dir."""
candidates = []
# The built-in argument parsing is not very good.
# Let's try our own.
try:
args = shlex.split(line)
except ValueError: # Catch badly formatted strings
args = line.split()
if len(args) == 1:
# Just the main command
# Text should be empty
search_text = ''
else:
search_text = args[-1]
text_offset = len(search_text) - len(text)
# Local commands start with 'l'.
# Special case 'ls'
if line[0] == 'l' and line[1] != 's':
# Local path
# Look further than just current dir
searchdir, searchfile = os.path.split(search_text)
abs_searchdir = searchdir
if not os.path.isabs(abs_searchdir):
abs_searchdir = os.path.join(self.localdir, abs_searchdir)
# Get contents of dir
for l in sh.ls(abs_searchdir, '-1', _iter=True):
l = l.strip()
if l.startswith(searchfile):
cand = os.path.join(searchdir, l)
if os.path.isdir(posixpath.join(abs_searchdir, l)):
cand += os.path.sep
candidates.append(cand[text_offset:])
else:
# Remote path
# Look further than just current dir
searchdir, searchfile = posixpath.split(search_text)
abs_searchdir = searchdir
if not posixpath.isabs(abs_searchdir):
abs_searchdir = posixpath.join(self.remotedir, abs_searchdir)
# Get contents of dir
for l in ls(abs_searchdir, _iter=True):
l = l.strip()
if l.startswith(searchfile):
cand = posixpath.join(searchdir, l)
if t2kdm.is_dir(posixpath.join(abs_searchdir, l),
cached=True):
cand += posixpath.sep
candidates.append(cand[text_offset:])
return candidates
def html_index(remotepath, localdir, recursive=False, topdir=False, verbose=False):
"""Generate a HTML index of the remote path in the local directory.
Returns the sum of the file sizes in the directory.
"""
if not os.path.isdir(localdir):
raise IOError("No such directory.")
if verbose:
print_("Creating index for %s..."%(remotepath,))
with temp_dir() as tempdir:
index_name = os.path.join(tempdir, "index.html")
size = 0
maxsize = 1
with open(index_name, 'wt') as f:
f.write("<!DOCTYPE html><html><head><title>%s</title></head><body><h3>%s</h3><table>\n"%(remotepath,remotepath))
f.write("<tr><th>size</th><th>modified</th><th>name</th></tr>\n")
if topdir:
# link to dir one level up
f.write("<tr><td style=\"text-align:right;\">-</td><td>-</td><td><a href=\"../index.html\">../</a></td></tr>\n")
for entry in dm.ls(remotepath):
path = posixpath.join(remotepath, entry.name)
if dm.is_dir(path):
if recursive:
newdir = os.path.join(localdir, entry.name)
try:
os.mkdir(newdir)
except OSError:
# directory probably exists
pass
sub_size = html_index(path, newdir, recursive=True, topdir=True, verbose=verbose)
f.write("<tr><td style=\"text-align:right;%s\">%s</td><td>%s</td><td><a href=\"%s/index.html\">%s/</a></td></tr>\n"%(_bgstyle(sub_size), _format_number(sub_size), entry.modified, entry.name, entry.name))
size += sub_size
maxsize = max(maxsize, sub_size)
else:
f.write("<tr><td style=\"text-align:right;\">%s</td><td>%s</td><td>%s/</td></tr>\n"%(_format_number(entry.size), entry.modified, entry.name))
else:
# Not a dir
f.write("<tr><td style=\"text-align:right;%s\">%s</td><td>%s</td><td>%s</td></tr>\n"%(_bgstyle(entry.size), _format_number(entry.size), entry.modified, entry.name))
if entry.size > 0:
size += entry.size
maxsize = max(maxsize, entry.size)
f.write("</table><p>Total size: %s</p><style>:root{--maxsize: %d} td,th{padding-left:3pt; padding-right:3pt;}</style></body></html>\n"%(_format_number(size),maxsize))
# Move file over
sh.mv(index_name, localdir, _tty_out=False)
return size
def do_cd(self, arg):
"""usage: cd remotepath
Change the current remote directory.
"""
pwd = self.get_abs_remote_path(arg)
# Let us see whether the path exists
try:
ls(pwd)
except t2kdm.backends.DoesNotExistException as e:
print_(e)
else:
# And whether it is a directory
if t2kdm.is_dir(pwd, cached=True):
self.remotedir = pwd
else:
print_("ERROR, not a directory: %s" % (pwd, ))
def remote_iter_recursively(remotepath, regex=None):
"""Iter over remote paths recursively.
If `regex` is given, only consider files/folders that match the reular expression.
"""
if isinstance(regex, str):
regex = re.compile(regex)
if t2kdm.is_dir(remotepath):
for entry in t2kdm.ls(remotepath):
if regex is None or regex.search(entry.name):
new_path = posixpath.join(remotepath, entry.name)
for path in remote_iter_recursively(new_path, regex):
yield path
else:
yield remotepath
def run_read_only_tests(tape=False, parallel=2):
print_("Testing ls...")
entries = dm.backend.ls(testdir)
for e in entries:
if e.name == testfiles[0]:
break
else:
raise Exception("Test file not in listing.")
print_("Testing ls_se...")
entries = dm.backend.ls_se(testdir, se=testSEs[0])
for e in entries:
if e.name == testfiles[0]:
break
else:
raise Exception("Test file not in listing.")
print_("Testing is_dir...")
assert (dm.is_dir(testdir))
print_("Testing is_dir_se...")
assert (dm.is_dir_se(testdir, se=testSEs[0]))
print_("Testing replicas...")
for rep in dm.backend.replicas(testpaths[0]):
if 'heplnx204.pp.rl.ac.uk' in rep:
break
else:
raise Exception("Did not find expected replica.")
print_("Testing iter_file_sources...")
for rep, se in dm.iter_file_sources(testpaths[0]):
if 'heplnx204.pp.rl.ac.uk' in rep:
break
else:
raise Exception("Did not find expected replica.")
print_("Testing is_file...")
assert (dm.backend.is_file(testpaths[0]))
assert (not dm.backend.is_file(testpaths[0] + "DNE"))
print_("Testing is_file_se...")
assert (dm.backend.is_file_se(testpaths[0], testSEs[0]))
assert (not dm.backend.is_file_se(testpaths[0] + "DNE", testSEs[0]))
print_("Testing exists...")
assert (dm.backend.exists(rep))
assert (not dm.backend.exists(posixpath.dirname(rep)))
print_("Testing checksum...")
assert (dm.backend.checksum(rep) == '529506c1')
print_("Testing state...")
assert ('ONLINE' in dm.backend.state(rep))
print_("Testing is_online...")
assert (dm.backend.is_online(rep))
print_("Testing StorageElement...")
# Test distance calculation
assert (storage.SEs[0].get_distance(storage.SEs[1]) < 0)
# Test getting SE by host
host = storage.SEs[0].host
assert (host in storage.SE_by_host[host].get_storage_path('/test'))
# Test storage path arithmetic
assert (storage.SEs[0].get_logical_path(
storage.SEs[0].get_storage_path('/test')) == '/test')
# Test getting the closest SE
assert (storage.get_closest_SE(testpaths[0]) is not None)
print_("Testing get...")
with temp_dir() as tempdir:
path = testpaths[0]
filename = os.path.join(tempdir, testfiles[0])
# Test choosing source SE automatically
assert (dm.backend.get(path, tempdir) == True)
assert (os.path.isfile(filename))
# Test tape refusal
try:
with no_output():
dm.backend.get(testpaths[2], tempdir)
except backends.BackendException as e:
assert ("Could not find" in e.args[0])
else:
raise Exception("Should have refused to download from tape!")
# Test providing the source SE (RAL tape!)
if tape:
print_("From tape!")
source = testSEs[2]
else:
source = testSEs[0]
try:
dm.backend.get(path, tempdir, source=source, force=False)
except backends.BackendException as e:
assert ("already exist" in e.args[0])
else:
raise Exception("Should have refused to overwrite!")
assert (dm.backend.get(path, tempdir, source=source,
force=True) == True)
assert (os.path.isfile(filename))
os.remove(filename)
# Test recursive get
assert (dm.interactive.get(testdir,
tempdir,
recursive='test[12]\.txt',
parallel=parallel) == 0)
assert (os.path.isfile(filename))
print_("Testing check...")
with temp_dir() as tempdir:
filename = os.path.join(tempdir, 'faulty.txt')
with no_output(True):
assert (dm.interactive.check(testdir,
checksum=True,
se=testSEs,
recursive=True,
quiet=False,
verbose=True,
list=filename,
parallel=parallel) != 0
) # There are some deliberate failures here
assert os.path.isfile(filename)
assert os.path.getsize(filename) > 0
with no_output(True):
assert (dm.interactive.check(testdir,
se=testSEs[0:1],
recursivese=testSEs[0],
quiet=False,
verbose=True) == 0)
print_("Testing HTML index...")
with temp_dir() as tempdir:
utils.html_index("/test/", tempdir)
utils.html_index("/test/", tempdir, recursive=True)
print_("Testing Commands...")
with no_output(True):
assert (cmd.ls.run_from_cli('-l /') == False)
assert (cmd.ls.run_from_cli('/', _return=True) == 0)
assert (cmd.ls.run_from_cli('abc') == False
) # This should not work, but not throw exception
assert (cmd.ls.run_from_cli('"abc', _return=True) != 0
) # This should not work, but not throw exception
with fake_argv(['%s-ls' % (dm._branding, ), '/']):
assert (cmd.ls.run_from_console() == 0) # This should work
with fake_argv(['%s-cli' % (dm._branding), '/']):
assert (cmd.ls.run_from_console() == 0) # This should work
with fake_argv(['%s-ls' % (dm._branding), '/abcxyz']):
assert (cmd.ls.run_from_console() != 0
) # This should not work, hence the not 0 return value
# None of the Commands should return True in the CLI
for com in cmd.all_commands:
assert (com.run_from_cli('') == False)
print_("Testing CLI...")
cli = dm.cli.T2KDmCli()
with no_output(True):
cli.onecmd('help ls')
cli.onecmd('ls .')
cli.onecmd('cd /test')
cli.onecmd('cd ..')
cli.onecmd('cd /abcxyz')
cli.onecmd('lcd /')
cli.onecmd('lcd .')
cli.onecmd('lcd /root')
cli.onecmd('lcd /abcxyz')
cli.onecmd('lls .')
cli.onecmd('lls ".')
cli.onecmd('lcd /')
assert (cli.completedefault('28', 'ls nd28', 0, 0) == ['280/'])
assert (cli.completedefault('s', 'lls us', 0, 0) == ['sr/'])
assert (cli.completedefault('"us', 'lls "us', 0, 0) == [])
def remote_iter_recursively(remotepath, regex=None, se=None, ignore_exceptions=False):
"""Iter over remote paths recursively.
If `regex` is given, only consider files/folders that match the reular expression.
If `se` is given, iterate over listing of physical files on SE rather than the file catalogue.
If `ignore_exceptions` is `True`, exceptions are ignored where possible.
"""
if isinstance(regex, str):
regex = re.compile(regex)
# Check whther the path is a directory.
# Check both in the file catalogue and on the storage element,
# because the directory might not yet exist in the catalogue.
for i in range(3):
# Try three times
try:
isdir = dm.is_dir(remotepath, cached=True) or (se is not None and dm.is_dir_se(remotepath, se, cached=True))
except Exception as e:
print_("Recursion failure! (%d)"%(i,))
if ignore_exceptions:
print_(e)
else:
raise
else:
# Break loop if no exception was raised (success)
break
sleep(10)
else:
# Return if loop was not broken
return
if isdir:
for i in range(3):
# Try three times
try:
if se is None:
entries = dm.iter_ls(remotepath)
else:
entries = dm.iter_ls_se(remotepath, se)
except Exception as e:
print_("Recursion failure! (%d)"%(i,))
if ignore_exceptions:
print_(e)
else:
raise
return
else:
# Break loop if no exception was raised (success)
break
sleep(10)
else:
# Return if loop was not broken
return
for entry in entries:
if regex is None or regex.search(entry.name):
new_path = posixpath.join(remotepath, entry.name)
for path in remote_iter_recursively(new_path, regex, se=se, ignore_exceptions=ignore_exceptions):
yield path
else:
yield str(remotepath)