def get_closest_SEs(self, remotepath=None, tape=False, cached=False):
"""Get a list of the storage element with the closest replicas.
If `tape` is False (default), prefer disk SEs over tape SEs.
If no `rempotepath` is provided, just return the closest SE over all.
"""
closest_SE = None
closest_distance = None
if remotepath is None:
candidates = SEs
else:
candidates = []
for rep in t2kdm.replicas(remotepath, cached=cached):
cand = get_SE_by_path(rep)
if cand is not None:
candidates.append(cand)
def sorter(SE):
if SE is None:
return 1000
distance = self.get_distance(SE)
if SE.type == 'tape':
if tape:
distance += 0.5
else:
distance += 10
if SE.is_blacklisted():
distance += 100
return distance
return sorted(candidates, key=sorter)
def get_replica(self, remotepath, cached=False):
"""Return the replica of the file on this SM."""
for rep in t2kdm.replicas(remotepath, cached=cached):
if self.host in rep:
return rep.strip()
# Replica not found
return None
def fix_known_bad_SEs(remotepath, verbose=False):
"""Fix replicas on known bad storage elements.
Unregisters all replicas on
IN2P3-CC-disk
"""
success = True
replicas = t2kdm.replicas(remotepath)
for replica in replicas:
se = storage.get_SE(replica)
if se is None:
print_("Found replica on unknown storage element: " + replica)
success = False
elif se.broken:
if verbose:
print_(
"Found replica on bad storage element. Unregistering replica: "
+ replica)
try:
t2kdm.backend.unregister(replica, remotepath, verbose=verbose)
except backends.BackendException():
if verbose:
print_("Failed to unregister replica.")
success = False
return success
def has_replica(self, remotepath, cached=False, check_dark=False):
"""Check whether the remote path is replicated on this SE.
If `check_dark` is `True`, check the physical file location, instead of relying on the catalogue.
"""
if not check_dark:
return any(self.host in replica
for replica in dm.replicas(remotepath, cached=cached))
else:
return dm.is_file_se(remotepath, self, cached=cached)
def check_replica_states(remotepath, cached=False):
"""Check if the state of all replicas."""
replicas = dm.replicas(remotepath, cached=cached)
for rep in replicas:
if dm.state(rep, cached=cached) not in ['ONLINE', 'NEARLINE', 'ONLINE_AND_NEARLINE']:
return False
return True
def check_checksums(remotepath, cached=False):
"""Check if the checksums of all replicas are identical."""
replicas = t2kdm.replicas(remotepath, cached=cached)
checksum = t2kdm.checksum(replicas[0], cached=cached)
if '?' in checksum:
return False
for rep in replicas[1:]:
if t2kdm.checksum(rep, cached=cached) != checksum:
return False
return True
def replicas(*args, **kwargs):
"""Print the replicas of a file on screen."""
checksum = kwargs.pop('checksum', False)
state = kwargs.pop('state', False)
name = kwargs.pop('name', False)
reps = t2kdm.replicas(*args, **kwargs)
for r in reps:
if checksum:
print_(t2kdm.checksum(r), end=' ')
if state:
print_(t2kdm.state(r), end=' ')
if name:
se = t2kdm.storage.get_SE(r)
if se is None:
print_('?', end=' ')
else:
print_(se.name, end=' ')
print_(r)
return 0
def replicas(remotepath, *args, **kwargs):
"""Print the replicas of a file on screen."""
_check_path(remotepath)
checksum = kwargs.pop('checksum', False)
state = kwargs.pop('state', False)
name = kwargs.pop('name', False)
distance = kwargs.pop('distance', False)
if distance:
if isinstance(distance, str):
reps = [
x[0] for x in dm.iter_file_sources(
remotepath, destination=distance, tape=True)
]
else:
reps = [x[0] for x in dm.iter_file_sources(remotepath, tape=True)]
else:
reps = dm.replicas(remotepath, *args, **kwargs)
for r in reps:
if checksum:
try:
chk = dm.checksum(r)
except Exception as e:
chk = str(e)
print_(chk, end=' ')
if state:
try:
stat = dm.state(r)
except Exception as e:
stat = str(e)
print_(stat, end=' ')
if name:
se = dm.storage.get_SE(r)
if se is None:
print_('?', end=' ')
else:
print_(se.name, end=' ')
print_(r)
return 0
def get_closest_SEs(self, remotepath=None, tape=False, cached=False):
"""Get a list of the storage element with the closest replicas.
If `tape` is False (default), do not return any tape SEs.
If no `rempotepath` is provided, just return the closest SE over all.
"""
on_tape = False
if remotepath is None:
candidates = SEs
else:
candidates = []
for rep in dm.replicas(remotepath, cached=cached):
cand = get_SE_by_path(rep)
if cand is None:
continue
if (cand.type == 'tape') and (tape == False):
on_tape = True
continue
candidates.append(cand)
if len(candidates) == 0 and on_tape:
print_(
"WARNING: Replica only found on tape, but tape sources are not accepted!"
)
def sorter(SE):
if SE is None:
return 1000
distance = self.get_distance(SE)
if SE.type == 'tape':
# Prefer disks over tape, even if the tape is closer by
distance += 10
if SE.is_blacklisted():
# Try blacklisted SEs only as a last resort
distance += 100
return distance
return sorted(candidates, key=sorter)
def has_replica(self, remotepath, cached=False):
"""Check whether the remote path is replicated on this SE."""
return any(self.host in replica for replica in t2kdm.replicas(remotepath, cached=cached))
def fix_missing_files(remotepath, verbose=False):
"""Fix missing files on the storage elements.
Helps when a replica is registered in the catalogue, but not actually present on the SE.
"""
success = True
replicas = t2kdm.replicas(remotepath)
existing = []
for rep in replicas:
se = storage.get_SE(rep)
if se is not None and se.is_blacklisted():
if verbose:
print_("WARNING: Skipping replica on blacklisted SE: " + rep)
print_("Will assume it exists for now.")
exists = True
success = False
else:
try:
exists = t2kdm.exists(rep)
except backends.BackendException:
if verbose:
print_(
"WARNING: Could not check whether replica exists: " +
rep)
print_("Will assume it does for now.")
exists = True
success = False
existing.append(exists)
# Check that there is at least one replica actually present
if not any(existing):
if verbose:
print_("WARNING: There is not a single replica actually present!")
print_("Doing nothing.")
return False
# Remove the replicas that are not present and remember which SEs those were
ses = []
for replica, exists in zip(replicas, existing):
if not exists:
if verbose:
print_("Found missing file. Unregistering replica: " + replica)
se = storage.get_SE(replica)
if se is not None:
ses.append(se)
try:
t2kdm.backend.unregister(replica,
remotepath,
verbose=verbose)
except backends.BackendException():
if verbose:
print_("Failed to unregister replica.")
success = False
else:
print_("Cannot identify storage element of replica.")
success = False
# Replicate the file
for se in ses:
if verbose:
print_("Replicating missing replica on " + se.name)
try:
t2kdm.replicate(remotepath, se, verbose=verbose)
except backends.BackendException():
if verbose:
print_("Failed to replicate File.")
success = False
return success
def fix_checksum_errors(remotepath, verbose=False):
"""Fix replicas with differing checksums.
This can only be done for files that can be checked for corruption.
Otherwise there is no way to decide which file is actually the correct one.
"""
replicas = dm.replicas(remotepath)
checksums = [dm.checksum(r) for r in replicas]
if len(set(checksums)) == 1 and '?' not in checksums[0]:
# Nothing to do here
return True
if verbose:
print_("Found faulty checksums.")
if not remotepath.endswith('.gz'):
if verbose:
print_("WARNING: Can only check file consistency of *.gz files!")
print_("Doing nothing.")
return False
good_replicas = []
bad_replicas = []
for replica in replicas:
if _test_replica(replica, verbose=verbose):
good_replicas.append(replica)
else:
bad_replicas.append(replica)
if len(good_replicas) == 0:
if verbose:
print_("WARNING: Not a single good replica present!")
print_("Doing nothing.")
return False
if len(bad_replicas) == 0:
if verbose:
print_("WARNING: Not a single bad replica present!")
print_("This should not happen, since the checksums are different.")
print_("Doing nothing.")
return False
bad_SEs = []
for replica in bad_replicas:
SE = storage.get_SE(replica)
if SE is None:
if verbose:
print_("WARNING: Could not find storage element for replica: "+replica)
continue
bad_SEs.append(SE)
success = True
for SE in bad_SEs:
if verbose:
print_("Removing bad replica from %s."%(SE.name,))
try:
dm.remove(remotepath, SE, verbose=verbose)
except:
success = False
for SE in bad_SEs:
if verbose:
print_("Re-replicating file on %s."%(SE.name,))
try:
dm.replicate(remotepath, SE, verbose=verbose)
except:
success = False
return success