def __init__(self):
self.did_client = DIDClient()
self.replica_client = ReplicaClient()
def test_list_dataset_replicas_archive(self):
""" REPLICA (CLIENT): List dataset replicas with archives. """
replica_client = ReplicaClient()
did_client = DIDClient()
rule_client = RuleClient()
scope = 'mock'
rse = 'APERTURE_%s' % rse_name_generator()
rse_id = add_rse(rse, **self.vo)
add_protocol(rse_id=rse_id,
parameter={
'scheme': 'root',
'hostname': 'root.aperture.com',
'port': 1409,
'prefix': '//test/chamber/',
'impl': 'rucio.rse.protocols.xrootd.Default',
'domains': {
'lan': {
'read': 1,
'write': 1,
'delete': 1
},
'wan': {
'read': 1,
'write': 1,
'delete': 1
}
}
})
rse2 = 'BLACKMESA_%s' % rse_name_generator()
rse2_id = add_rse(rse2, **self.vo)
add_protocol(rse_id=rse2_id,
parameter={
'scheme': 'root',
'hostname': 'root.blackmesa.com',
'port': 1409,
'prefix': '//underground/facility',
'impl': 'rucio.rse.protocols.xrootd.Default',
'domains': {
'lan': {
'read': 1,
'write': 1,
'delete': 1
},
'wan': {
'read': 1,
'write': 1,
'delete': 1
}
}
})
# register archive
archive = {
'scope': scope,
'name': 'another.%s.zip' % generate_uuid(),
'type': 'FILE',
'bytes': 2596,
'adler32': 'deedbeaf'
}
replica_client.add_replicas(rse=rse, files=[archive])
replica_client.add_replicas(rse=rse2, files=[archive])
archived_files = [{
'scope':
scope,
'name':
'zippedfile-%i-%s' % (i, str(generate_uuid())),
'type':
'FILE',
'bytes':
4322,
'adler32':
'deaddead'
} for i in range(2)]
replica_client.add_replicas(rse=rse2, files=archived_files)
did_client.add_files_to_archive(scope=scope,
name=archive['name'],
files=archived_files)
dataset_name = 'find_me.' + str(generate_uuid())
did_client.add_dataset(scope=scope, name=dataset_name)
did_client.attach_dids(scope=scope,
name=dataset_name,
dids=archived_files)
rule_client.add_replication_rule(dids=[{
'scope': scope,
'name': dataset_name
}],
account='root',
copies=1,
rse_expression=rse,
grouping='DATASET')
res = [
r for r in replica_client.list_dataset_replicas(scope=scope,
name=dataset_name)
]
assert len(res) == 1
assert res[0]['state'] == 'UNAVAILABLE'
res = [
r for r in replica_client.list_dataset_replicas(
scope=scope, name=dataset_name, deep=True)
]
assert len(res) == 3
assert res[0]['state'] == 'AVAILABLE'
assert res[1]['state'] == 'AVAILABLE'
assert res[2]['state'] == 'AVAILABLE'
del_rse(rse_id)
def __init__(self):
self.dc = DIDClient()
self.rc = ReplicaClient()
def setup(self):
self.replica_client = ReplicaClient()
self.did_client = DIDClient()
try:
from StringIO import StringIO ## for Python 2
except ImportError:
from io import StringIO ## for Python 3
import linecache
# Rucio settings
rulesClient = RuleClient()
account = 'root'
auth_type = 'x509_proxy'
# account=account, auth_type=auth_type
CLIENT = Client(account=account)
didc = DIDClient(account=account)
repc = ReplicaClient(account=account)
rulesClient = RuleClient(account=account)
client = Client(account=account)
uploadClient = UploadClient()
print(client.whoami())
print(client.ping())
sys.path.append("/usr/lib64/python3.6/site-packages/")
import gfal2
from gfal2 import Gfal2Context, GError
gfal = Gfal2Context()
# Global variables definition :
if PROTO['port'] != 0:
URL = URL + ':' + str(PROTO['port'])
URL = URL + PREFIX + '/' + OPTIONS.name
GFAL = Gfal2Context()
try:
SIZE = GFAL.stat(str(URL)).st_size
CHECKSUM = GFAL.checksum(str(URL), 'adler32')
print("Registering file: pfn=%s size=%s checksum=%s" %
(URL, SIZE, CHECKSUM))
except GError:
print("no file found at %s" % URL)
exit()
R = ReplicaClient()
REPLICAS = list(
R.list_replicas([{
'scope': OPTIONS.scope,
'name': OPTIONS.name
}]))
if REPLICAS:
REPLICAS = REPLICAS[0]
if 'rses' in REPLICAS:
if OPTIONS.rse in REPLICAS['rses']:
print("file %s with scope %s has already a replica at %s" %
(OPTIONS.name, OPTIONS.scope, OPTIONS.rse))
exit()
REPLICA = [{
def test_replica_no_site(self):
""" REPLICA (CORE): Test listing replicas without site attribute """
rc = ReplicaClient()
rse = 'APERTURE_%s' % rse_name_generator()
add_rse(rse)
add_protocol(
rse, {
'scheme': 'root',
'hostname': 'root.aperture.com',
'port': 1409,
'prefix': '//test/chamber/',
'impl': 'rucio.rse.protocols.xrootd.Default',
'domains': {
'lan': {
'read': 1,
'write': 1,
'delete': 1
},
'wan': {
'read': 1,
'write': 1,
'delete': 1
}
}
})
add_rse_attribute(rse=rse, key='site', value='APERTURE')
files = [{
'scope': 'mock',
'name': 'element_%s' % generate_uuid(),
'bytes': 1234,
'adler32': 'deadbeef'
}]
add_replicas(rse=rse, files=files, account='root')
replicas = [
r for r in rc.list_replicas(dids=[{
'scope': 'mock',
'name': f['name']
} for f in files])
]
assert_in('root://', replicas[0]['pfns'].keys()[0])
replicas = [
r for r in rc.list_replicas(dids=[{
'scope': 'mock',
'name': f['name']
} for f in files],
client_location={'site': 'SOMEWHERE'})
]
assert_in('root://', replicas[0]['pfns'].keys()[0])
del_rse_attribute(rse=rse, key='site')
replicas = [
r for r in rc.list_replicas(dids=[{
'scope': 'mock',
'name': f['name']
} for f in files])
]
assert_in('root://', replicas[0]['pfns'].keys()[0])
replicas = [
r for r in rc.list_replicas(dids=[{
'scope': 'mock',
'name': f['name']
} for f in files],
client_location={'site': 'SOMEWHERE'})
]
assert_in('root://', replicas[0]['pfns'].keys()[0])
def setUp(self):
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
self.vo = {'vo': get_vo()}
else:
self.vo = {}
self.replica_client = ReplicaClient()
assert hasattr(self, "rse_factory")
rse_factory = self.rse_factory
# Using two test RSEs
self.rse4suspicious, self.rse4suspicious_id = rse_factory.make_posix_rse(
deterministic=True, **self.vo)
self.rse4recovery, self.rse4recovery_id = rse_factory.make_posix_rse(
deterministic=True, **self.vo)
self.scope = 'mock'
self.internal_scope = InternalScope(self.scope, **self.vo)
# For testing, we create 5 files and upload them to Rucio to two test RSEs.
self.tmp_file1 = file_generator()
self.tmp_file2 = file_generator()
self.tmp_file3 = file_generator()
self.tmp_file4 = file_generator()
self.tmp_file5 = file_generator()
self.listdids = [{
'scope': self.internal_scope,
'name': path.basename(f),
'type': DIDType.FILE
} for f in [
self.tmp_file1, self.tmp_file2, self.tmp_file3, self.tmp_file4,
self.tmp_file5
]]
for rse in [self.rse4suspicious, self.rse4recovery]:
cmd = 'rucio -v upload --rse {0} --scope {1} {2} {3} {4} {5} {6}'.format(
rse, self.scope, self.tmp_file1, self.tmp_file2,
self.tmp_file3, self.tmp_file4, self.tmp_file5)
exitcode, out, err = execute(cmd)
print(exitcode, out, err)
# checking if Rucio upload went OK
assert exitcode == 0
# Set fictional datatypes
set_metadata(self.internal_scope, path.basename(self.tmp_file4),
'datatype', 'testtypedeclarebad')
set_metadata(self.internal_scope, path.basename(self.tmp_file5),
'datatype', 'testtypenopolicy')
# Allow for the RSEs to be affected by the suspicious file recovery daemon
add_rse_attribute(self.rse4suspicious_id,
"enable_suspicious_file_recovery", True)
add_rse_attribute(self.rse4recovery_id,
"enable_suspicious_file_recovery", True)
# removing physical files from /tmp location - keeping only their DB info
remove(self.tmp_file1)
remove(self.tmp_file2)
remove(self.tmp_file3)
remove(self.tmp_file4)
remove(self.tmp_file5)
# Gather replica info
replicalist = list_replicas(dids=self.listdids)
# Changing the replica statuses as follows:
# ----------------------------------------------------------------------------------------------------------------------------------
# Name State(s) declared on rse4recovery State(s) declared on rse4suspicious Metadata "datatype"
# ----------------------------------------------------------------------------------------------------------------------------------
# tmp_file1 available suspicious (available)
# tmp_file2 available suspicious + bad (unavailable)
# tmp_file3 unavailable suspicious (available) RAW
# tmp_file4 unavailable suspicious (available) testtypedeclarebad
# tmp_file5 unavailable suspicious (available) testtypenopolicy
# ----------------------------------------------------------------------------------------------------------------------------------
for replica in replicalist:
suspicious_pfns = replica['rses'][self.rse4suspicious_id]
for i in range(3):
print("Declaring suspicious file replica: " +
suspicious_pfns[0])
# The reason must contain the word "checksum", so that the replica can be declared bad.
self.replica_client.declare_suspicious_file_replicas([
suspicious_pfns[0],
], 'checksum')
sleep(1)
if replica['name'] == path.basename(self.tmp_file2):
print("Declaring bad file replica: " + suspicious_pfns[0])
self.replica_client.declare_bad_file_replicas([
suspicious_pfns[0],
], 'checksum')
if replica['name'] == path.basename(self.tmp_file3):
print("Updating replica state as unavailable: " +
replica['rses'][self.rse4recovery_id][0])
update_replica_state(self.rse4recovery_id, self.internal_scope,
path.basename(self.tmp_file3),
ReplicaState.UNAVAILABLE)
if replica['name'] == path.basename(self.tmp_file4):
print("Updating replica state as unavailable: " +
replica['rses'][self.rse4recovery_id][0])
update_replica_state(self.rse4recovery_id, self.internal_scope,
path.basename(self.tmp_file4),
ReplicaState.UNAVAILABLE)
if replica['name'] == path.basename(self.tmp_file5):
print("Updating replica state as unavailable: " +
replica['rses'][self.rse4recovery_id][0])
update_replica_state(self.rse4recovery_id, self.internal_scope,
path.basename(self.tmp_file5),
ReplicaState.UNAVAILABLE)
# Gather replica info after setting initial replica statuses
replicalist = list_replicas(dids=self.listdids)
# Checking if the status changes were effective
for replica in replicalist:
if replica['name'] == path.basename(self.tmp_file1):
assert replica['states'][self.rse4suspicious_id] == 'AVAILABLE'
assert replica['states'][self.rse4recovery_id] == 'AVAILABLE'
if replica['name'] == path.basename(self.tmp_file2):
assert (self.rse4suspicious_id in replica['states']) is False
assert replica['states'][self.rse4recovery_id] == 'AVAILABLE'
if replica['name'] == path.basename(self.tmp_file3):
assert replica['states'][self.rse4suspicious_id] == 'AVAILABLE'
assert (self.rse4recovery_id in replica['states']) is False
if replica['name'] == path.basename(self.tmp_file4):
assert replica['states'][self.rse4suspicious_id] == 'AVAILABLE'
assert (self.rse4recovery_id in replica['states']) is False
if replica['name'] == path.basename(self.tmp_file5):
assert replica['states'][self.rse4suspicious_id] == 'AVAILABLE'
assert (self.rse4recovery_id in replica['states']) is False
# Checking if only self.tmp_file2 is declared as 'BAD'
self.from_date = datetime.now() - timedelta(days=1)
bad_replicas_list = list_bad_replicas_status(
rse_id=self.rse4suspicious_id,
younger_than=self.from_date,
**self.vo)
bad_checklist = [(badf['name'], badf['rse_id'], badf['state'])
for badf in bad_replicas_list]
assert (path.basename(self.tmp_file1), self.rse4suspicious_id,
BadFilesStatus.BAD) not in bad_checklist
assert (path.basename(self.tmp_file2), self.rse4suspicious_id,
BadFilesStatus.BAD) in bad_checklist
assert (path.basename(self.tmp_file3), self.rse4suspicious_id,
BadFilesStatus.BAD) not in bad_checklist
assert (path.basename(self.tmp_file4), self.rse4suspicious_id,
BadFilesStatus.BAD) not in bad_checklist
assert (path.basename(self.tmp_file5), self.rse4suspicious_id,
BadFilesStatus.BAD) not in bad_checklist
bad_replicas_list = list_bad_replicas_status(
rse_id=self.rse4recovery_id,
younger_than=self.from_date,
**self.vo)
bad_checklist = [(badf['name'], badf['rse_id'], badf['state'])
for badf in bad_replicas_list]
assert (path.basename(self.tmp_file1), self.rse4recovery_id,
BadFilesStatus.BAD) not in bad_checklist
assert (path.basename(self.tmp_file2), self.rse4recovery_id,
BadFilesStatus.BAD) not in bad_checklist
assert (path.basename(self.tmp_file3), self.rse4recovery_id,
BadFilesStatus.BAD) not in bad_checklist
assert (path.basename(self.tmp_file4), self.rse4recovery_id,
BadFilesStatus.BAD) not in bad_checklist
assert (path.basename(self.tmp_file5), self.rse4recovery_id,
BadFilesStatus.BAD) not in bad_checklist
def setUp(self):
self.replica_client = ReplicaClient()
# Using two test RSEs
self.rse4suspicious = 'MOCK_SUSPICIOUS'
self.rse4recovery = 'MOCK_RECOVERY'
self.scope = 'mock'
# For testing, we create 3 files and upload them to Rucio to two test RSEs.
self.tmp_file1 = file_generator()
self.tmp_file2 = file_generator()
self.tmp_file3 = file_generator()
self.listdids = [{
'scope': self.scope,
'name': path.basename(self.tmp_file1),
'type': DIDType.FILE
}, {
'scope': self.scope,
'name': path.basename(self.tmp_file2),
'type': DIDType.FILE
}, {
'scope': self.scope,
'name': path.basename(self.tmp_file3),
'type': DIDType.FILE
}]
for rse in [self.rse4suspicious, self.rse4recovery]:
cmd = 'rucio -v upload --rse {0} --scope {1} {2} {3} {4}'.format(
rse, self.scope, self.tmp_file1, self.tmp_file2,
self.tmp_file3)
exitcode, out, err = execute(cmd)
# checking if Rucio upload went OK
assert_true(exitcode == 0)
# removing physical files from /tmp location - keeping only their DB info
remove(self.tmp_file1)
remove(self.tmp_file2)
remove(self.tmp_file3)
# Gather replica info
replicalist = list_replicas(dids=self.listdids)
# Changing the replica statuses as follows:
# --------------------------------------------------------------------------------------------
# Name State(s) declared on MOCK_RECOVERY State(s) declared on MOCK_SUSPICIOUS
# --------------------------------------------------------------------------------------------
# tmp_file1 available suspicious (available)
# tmp_file2 available suspicious + bad (unavailable)
# tmp_file3 unavailable suspicious (available)
# --------------------------------------------------------------------------------------------
for replica in replicalist:
for i in range(3):
print("Declaring suspicious file replica: " +
replica['rses'][self.rse4suspicious][0])
self.replica_client.declare_suspicious_file_replicas([
replica['rses'][self.rse4suspicious][0],
], 'This is a good reason.')
sleep(1)
if replica['name'] == path.basename(self.tmp_file2):
print("Declaring bad file replica: " +
replica['rses'][self.rse4suspicious][0])
self.replica_client.declare_bad_file_replicas([
replica['rses'][self.rse4suspicious][0],
], 'This is a good reason')
if replica['name'] == path.basename(self.tmp_file3):
print("Updating replica state as unavailable: " +
replica['rses'][self.rse4recovery][0])
update_replica_state(self.rse4recovery, self.scope,
path.basename(self.tmp_file3),
ReplicaState.UNAVAILABLE)
# Gather replica info after setting initial replica statuses
replicalist = list_replicas(dids=self.listdids)
# Checking if the status changes were effective
for replica in replicalist:
if replica['name'] == path.basename(self.tmp_file1):
assert_true(
replica['states'][self.rse4suspicious] == 'AVAILABLE')
assert_true(
replica['states'][self.rse4recovery] == 'AVAILABLE')
if replica['name'] == path.basename(self.tmp_file2):
assert_true(
(self.rse4suspicious in replica['states']) is False)
assert_true(
replica['states'][self.rse4recovery] == 'AVAILABLE')
if replica['name'] == path.basename(self.tmp_file3):
assert_true(
replica['states'][self.rse4suspicious] == 'AVAILABLE')
assert_true((self.rse4recovery in replica['states']) is False)
# Checking if only self.tmp_file2 is declared as 'BAD'
self.from_date = datetime.now() - timedelta(days=1)
bad_replicas_list = list_bad_replicas_status(
rse=self.rse4suspicious, younger_than=self.from_date)
bad_checklist = [(badf['name'], badf['rse'], badf['state'])
for badf in bad_replicas_list]
assert_true((path.basename(self.tmp_file2), self.rse4suspicious,
BadFilesStatus.BAD) in bad_checklist)
assert_true((path.basename(self.tmp_file1), self.rse4suspicious,
BadFilesStatus.BAD) not in bad_checklist)
assert_true((path.basename(self.tmp_file3), self.rse4suspicious,
BadFilesStatus.BAD) not in bad_checklist)
bad_replicas_list = list_bad_replicas_status(
rse=self.rse4recovery, younger_than=self.from_date)
bad_checklist = [(badf['name'], badf['rse'], badf['state'])
for badf in bad_replicas_list]
assert_true((path.basename(self.tmp_file1), self.rse4recovery,
BadFilesStatus.BAD) not in bad_checklist)
assert_true((path.basename(self.tmp_file2), self.rse4recovery,
BadFilesStatus.BAD) not in bad_checklist)
assert_true((path.basename(self.tmp_file3), self.rse4recovery,
BadFilesStatus.BAD) not in bad_checklist)
def replica_client():
from rucio.client.replicaclient import ReplicaClient
return ReplicaClient()
def _rucio_register(beamline, uid, filenames):
"""
Register the file in rucio for replication to SDCC.
"""
scope = beamline
container = uid
replica_client = ReplicaClient()
didclient = DIDClient()
scopeclient = ScopeClient()
ruleclient = RuleClient()
for root, ending, filename in filenames:
#size = os.stat(str(filename)).st_size
#adler = adler32(str(filename))
files = [{
'scope': scope,
'name': filename.split('/')[-1],
'bytes': 1000,
#'adler32': "unknown",
'pfn': pfn + filename
}]
dataset = os.path.join(root, ending)
dataset = '.'.join(dataset.split('/')[1:-1])
print("DATASET", dataset)
breakpoint()
try:
scopeclient.add_scope(account='nsls2data', scope=scope)
except rucio.common.exception.Duplicate:
pass
replica_client.add_replicas(rse=rse, files=files)
# Create a new container if it doesn't exist.
try:
didclient.add_did(scope=scope, name=uid, type='container')
except rucio.common.exception.DataIdentifierAlreadyExists:
pass
# Create a replication rule.
try:
dids = [{'scope': scope, 'name': container}]
ruleclient.add_replication_rule(
dids=dids,
copies=1,
rse_expression='SDCC',
lifetime=86400, # Seconds
account='nsls2data',
source_replica_expression='NSLS2',
purge_replicas=True,
comment='purge_replicas in 24 hours')
except rucio.common.exception.DuplicateRule:
pass
# Create a new dataset if it doesn't exist.
try:
didclient.add_did(scope=scope, name=dataset, type='dataset')
except rucio.common.exception.DataIdentifierAlreadyExists:
pass
attachment = {
'scope': scope,
'name': uid,
'dids': [{
'scope': scope,
'name': dataset
}]
}
try:
didclient.add_files_to_dataset(scope, dataset, files)
except rucio.common.exception.FileAlreadyExists:
pass
try:
didclient.add_datasets_to_containers([attachment])
except rucio.common.exception.DuplicateContent:
pass
def test_replica_sorting(self):
""" REPLICA (CORE): Test the correct sorting of the replicas across WAN and LAN """
self.rc = ReplicaClient()
self.rse1 = 'APERTURE_%s' % rse_name_generator()
self.rse2 = 'BLACKMESA_%s' % rse_name_generator()
self.rse1_id = add_rse(self.rse1, **self.vo)
self.rse2_id = add_rse(self.rse2, **self.vo)
add_rse_attribute(rse_id=self.rse1_id, key='site', value='APERTURE')
add_rse_attribute(rse_id=self.rse2_id, key='site', value='BLACKMESA')
self.files = [{'scope': InternalScope('mock', **self.vo), 'name': 'element_0',
'bytes': 1234, 'adler32': 'deadbeef'}]
root = InternalAccount('root', **self.vo)
add_replicas(rse_id=self.rse1_id, files=self.files, account=root)
add_replicas(rse_id=self.rse2_id, files=self.files, account=root)
add_protocol(self.rse1_id, {'scheme': 'root',
'hostname': 'root.aperture.com',
'port': 1409,
'prefix': '//test/chamber/',
'impl': 'rucio.rse.protocols.xrootd.Default',
'domains': {
'lan': {'read': 1, 'write': 1, 'delete': 1},
'wan': {'read': 1, 'write': 1, 'delete': 1}}})
add_protocol(self.rse1_id, {'scheme': 'davs',
'hostname': 'davs.aperture.com',
'port': 443,
'prefix': '/test/chamber/',
'impl': 'rucio.rse.protocols.gfal.Default',
'domains': {
'lan': {'read': 2, 'write': 2, 'delete': 2},
'wan': {'read': 2, 'write': 2, 'delete': 2}}})
add_protocol(self.rse1_id, {'scheme': 'gsiftp',
'hostname': 'gsiftp.aperture.com',
'port': 8446,
'prefix': '/test/chamber/',
'impl': 'rucio.rse.protocols.gfal.Default',
'domains': {
'lan': {'read': 0, 'write': 0, 'delete': 0},
'wan': {'read': 3, 'write': 3, 'delete': 3}}})
add_protocol(self.rse2_id, {'scheme': 'gsiftp',
'hostname': 'gsiftp.blackmesa.com',
'port': 8446,
'prefix': '/lambda/complex/',
'impl': 'rucio.rse.protocols.gfal.Default',
'domains': {
'lan': {'read': 2, 'write': 2, 'delete': 2},
'wan': {'read': 1, 'write': 1, 'delete': 1}}})
add_protocol(self.rse2_id, {'scheme': 'davs',
'hostname': 'davs.blackmesa.com',
'port': 443,
'prefix': '/lambda/complex/',
'impl': 'rucio.rse.protocols.gfal.Default',
'domains': {
'lan': {'read': 0, 'write': 0, 'delete': 0},
'wan': {'read': 2, 'write': 2, 'delete': 2}}})
add_protocol(self.rse2_id, {'scheme': 'root',
'hostname': 'root.blackmesa.com',
'port': 1409,
'prefix': '//lambda/complex/',
'impl': 'rucio.rse.protocols.xrootd.Default',
'domains': {
'lan': {'read': 1, 'write': 1, 'delete': 1},
'wan': {'read': 3, 'write': 3, 'delete': 3}}})
replicas = [r for r in self.rc.list_replicas(dids=[{'scope': 'mock',
'name': f['name'],
'type': 'FILE'} for f in self.files],
schemes=['root', 'gsiftp', 'davs'],
client_location={'site': 'APERTURE'})]
pfns = [r['pfns'] for r in replicas][0]
assert len(pfns.keys()) == 5
assert pfns['root://root.aperture.com:1409//test/chamber/mock/58/b5/element_0']['domain'] == 'lan'
assert pfns['root://root.aperture.com:1409//test/chamber/mock/58/b5/element_0']['priority'] == 1
assert pfns['davs://davs.aperture.com:443/test/chamber/mock/58/b5/element_0']['domain'] == 'lan'
assert pfns['davs://davs.aperture.com:443/test/chamber/mock/58/b5/element_0']['priority'] == 2
assert pfns['gsiftp://gsiftp.blackmesa.com:8446/lambda/complex/mock/58/b5/element_0']['domain'] == 'wan'
assert pfns['gsiftp://gsiftp.blackmesa.com:8446/lambda/complex/mock/58/b5/element_0']['priority'] == 3
assert pfns['davs://davs.blackmesa.com:443/lambda/complex/mock/58/b5/element_0']['domain'] == 'wan'
assert pfns['davs://davs.blackmesa.com:443/lambda/complex/mock/58/b5/element_0']['priority'] == 4
assert pfns['root://root.blackmesa.com:1409//lambda/complex/mock/58/b5/element_0']['domain'] == 'wan'
assert pfns['root://root.blackmesa.com:1409//lambda/complex/mock/58/b5/element_0']['priority'] == 5
replicas = [r for r in self.rc.list_replicas(dids=[{'scope': 'mock',
'name': f['name'],
'type': 'FILE'} for f in self.files],
schemes=['root', 'gsiftp', 'davs'],
client_location={'site': 'BLACKMESA'})]
pfns = [r['pfns'] for r in replicas][0]
assert len(pfns.keys()) == 5
assert pfns['root://root.blackmesa.com:1409//lambda/complex/mock/58/b5/element_0']['domain'] == 'lan'
assert pfns['root://root.blackmesa.com:1409//lambda/complex/mock/58/b5/element_0']['priority'] == 1
assert pfns['gsiftp://gsiftp.blackmesa.com:8446/lambda/complex/mock/58/b5/element_0']['domain'] == 'lan'
assert pfns['gsiftp://gsiftp.blackmesa.com:8446/lambda/complex/mock/58/b5/element_0']['priority'] == 2
assert pfns['root://root.aperture.com:1409//test/chamber/mock/58/b5/element_0']['domain'] == 'wan'
assert pfns['root://root.aperture.com:1409//test/chamber/mock/58/b5/element_0']['priority'] == 3
assert pfns['davs://davs.aperture.com:443/test/chamber/mock/58/b5/element_0']['domain'] == 'wan'
assert pfns['davs://davs.aperture.com:443/test/chamber/mock/58/b5/element_0']['priority'] == 4
assert pfns['gsiftp://gsiftp.aperture.com:8446/test/chamber/mock/58/b5/element_0']['domain'] == 'wan'
assert pfns['gsiftp://gsiftp.aperture.com:8446/test/chamber/mock/58/b5/element_0']['priority'] == 5
replicas = [r for r in self.rc.list_replicas(dids=[{'scope': 'mock',
'name': f['name'],
'type': 'FILE'} for f in self.files],
schemes=['root', 'gsiftp', 'davs'],
client_location={'site': 'XEN'})]
pfns = [r['pfns'] for r in replicas][0]
assert len(pfns.keys()) == 6
# TODO: intractable until RSE sorting is enabled
assert pfns['gsiftp://gsiftp.blackmesa.com:8446/lambda/complex/mock/58/b5/element_0']['domain'] == 'wan'
assert pfns['gsiftp://gsiftp.blackmesa.com:8446/lambda/complex/mock/58/b5/element_0']['priority'] in [1, 2]
assert pfns['root://root.aperture.com:1409//test/chamber/mock/58/b5/element_0']['domain'] == 'wan'
assert pfns['root://root.aperture.com:1409//test/chamber/mock/58/b5/element_0']['priority'] in [1, 2]
assert pfns['davs://davs.aperture.com:443/test/chamber/mock/58/b5/element_0']['domain'] == 'wan'
assert pfns['davs://davs.aperture.com:443/test/chamber/mock/58/b5/element_0']['priority'] in [3, 4]
assert pfns['davs://davs.blackmesa.com:443/lambda/complex/mock/58/b5/element_0']['domain'] == 'wan'
assert pfns['davs://davs.blackmesa.com:443/lambda/complex/mock/58/b5/element_0']['priority'] in [3, 4]
assert pfns['gsiftp://gsiftp.aperture.com:8446/test/chamber/mock/58/b5/element_0']['domain'] == 'wan'
assert pfns['gsiftp://gsiftp.aperture.com:8446/test/chamber/mock/58/b5/element_0']['priority'] in [5, 6]
assert pfns['root://root.blackmesa.com:1409//lambda/complex/mock/58/b5/element_0']['domain'] == 'wan'
assert pfns['root://root.blackmesa.com:1409//lambda/complex/mock/58/b5/element_0']['priority'] in [5, 6]
ml = self.rc.list_replicas(dids=[{'scope': 'mock',
'name': f['name'],
'type': 'FILE'} for f in self.files],
schemes=['root', 'gsiftp', 'davs'],
metalink=True,
client_location={'site': 'APERTURE'})
assert 'domain="lan" priority="1" client_extract="false">root://root.aperture.com:1409//test/chamber/mock/58/b5/element_0' in ml
assert 'domain="lan" priority="2" client_extract="false">davs://davs.aperture.com:443/test/chamber/mock/58/b5/element_0' in ml
assert 'domain="wan" priority="3" client_extract="false">gsiftp://gsiftp.blackmesa.com:8446/lambda/complex/mock/58/b5/element_0' in ml
assert 'domain="wan" priority="4" client_extract="false">davs://davs.blackmesa.com:443/lambda/complex/mock/58/b5/element_0' in ml
assert 'domain="wan" priority="5" client_extract="false">root://root.blackmesa.com:1409//lambda/complex/mock/58/b5/element_0' in ml
assert 'priority="6"' not in ml
ml = self.rc.list_replicas(dids=[{'scope': 'mock',
'name': f['name'],
'type': 'FILE'} for f in self.files],
schemes=['root', 'gsiftp', 'davs'],
metalink=True,
client_location={'site': 'BLACKMESA'})
assert 'domain="lan" priority="1" client_extract="false">root://root.blackmesa.com:1409//lambda/complex/mock/58/b5/element_0' in ml
assert 'domain="lan" priority="2" client_extract="false">gsiftp://gsiftp.blackmesa.com:8446/lambda/complex/mock/58/b5/element_0' in ml
assert 'domain="wan" priority="3" client_extract="false">root://root.aperture.com:1409//test/chamber/mock/58/b5/element_0' in ml
assert 'domain="wan" priority="4" client_extract="false">davs://davs.aperture.com:443/test/chamber/mock/58/b5/element_0' in ml
assert 'domain="wan" priority="5" client_extract="false">gsiftp://gsiftp.aperture.com:8446/test/chamber/mock/58/b5/element_0' in ml
assert 'priority="6"' not in ml
# TODO: intractable until RSE sorting is enabled
# ml = self.rc.list_replicas(dids=[{'scope': 'mock',
# 'name': f['name'],
# 'type': 'FILE'} for f in self.files],
# schemes=['root', 'gsiftp', 'davs'],
# metalink=True,
# client_location={'site': 'XEN'})
# assert 'domain="wan" priority="1">root://root.aperture.com:1409//test/chamber/mock/58/b5/element_0' in ml
# assert 'domain="wan" priority="2">gsiftp://gsiftp.blackmesa.com:8446/lambda/complex/mock/58/b5/element_0' in ml
# assert 'domain="wan" priority="3">davs://davs.aperture.com:443/test/chamber/mock/58/b5/element_0' in ml
# assert 'domain="wan" priority="4">davs://davs.blackmesa.com:443/lambda/complex/mock/58/b5/element_0' in ml
# assert 'domain="wan" priority="5">gsiftp://gsiftp.aperture.com:8446/test/chamber/mock/58/b5/element_0' in ml
# assert 'domain="wan" priority="6">root://root.blackmesa.com:1409//lambda/complex/mock/58/b5/element_0' in ml
# assert 'priority="7"' not in ml
# ensure correct handling of disabled protocols
add_protocol(self.rse1_id, {'scheme': 'root',
'hostname': 'root2.aperture.com',
'port': 1409,
'prefix': '//test/chamber/',
'impl': 'rucio.rse.protocols.xrootd.Default',
'domains': {
'lan': {'read': 1, 'write': 1, 'delete': 1},
'wan': {'read': 0, 'write': 0, 'delete': 0}}})
ml = self.rc.list_replicas(dids=[{'scope': 'mock',
'name': f['name'],
'type': 'FILE'} for f in self.files],
schemes=['root', 'gsiftp', 'davs'],
metalink=True,
client_location={'site': 'BLACKMESA'})
assert 'domain="lan" priority="1" client_extract="false">root://root.blackmesa.com:1409//lambda/complex/mock/58/b5/element_0' in ml
assert 'domain="lan" priority="2" client_extract="false">gsiftp://gsiftp.blackmesa.com:8446/lambda/complex/mock/58/b5/element_0' in ml
assert 'domain="wan" priority="3" client_extract="false">root://root.aperture.com:1409//test/chamber/mock/58/b5/element_0' in ml
assert 'domain="wan" priority="4" client_extract="false">davs://davs.aperture.com:443/test/chamber/mock/58/b5/element_0' in ml
assert 'domain="wan" priority="5" client_extract="false">gsiftp://gsiftp.aperture.com:8446/test/chamber/mock/58/b5/element_0' in ml
assert 'priority="6"' not in ml
delete_replicas(rse_id=self.rse1_id, files=self.files)
delete_replicas(rse_id=self.rse2_id, files=self.files)
del_rse(self.rse1_id)
del_rse(self.rse2_id)
def setUp(self):
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
self.vo = {
'vo':
config_get('client',
'vo',
raise_exception=False,
default='tst')
}
else:
self.vo = {}
self.replica_client = ReplicaClient()
# Using two test RSEs
self.rse4suspicious = 'MOCK_SUSPICIOUS'
self.rse4suspicious_id = get_rse_id(self.rse4suspicious, **self.vo)
self.rse4recovery = 'MOCK_RECOVERY'
self.rse4recovery_id = get_rse_id(self.rse4recovery, **self.vo)
self.scope = 'mock'
self.internal_scope = InternalScope(self.scope, **self.vo)
# For testing, we create 3 files and upload them to Rucio to two test RSEs.
self.tmp_file1 = file_generator()
self.tmp_file2 = file_generator()
self.tmp_file3 = file_generator()
self.listdids = [{
'scope': self.internal_scope,
'name': path.basename(f),
'type': DIDType.FILE
} for f in [self.tmp_file1, self.tmp_file2, self.tmp_file3]]
for rse in [self.rse4suspicious, self.rse4recovery]:
cmd = 'rucio -v upload --rse {0} --scope {1} {2} {3} {4}'.format(
rse, self.scope, self.tmp_file1, self.tmp_file2,
self.tmp_file3)
exitcode, out, err = execute(cmd)
# checking if Rucio upload went OK
assert exitcode == 0
# removing physical files from /tmp location - keeping only their DB info
remove(self.tmp_file1)
remove(self.tmp_file2)
remove(self.tmp_file3)
# Gather replica info
replicalist = list_replicas(dids=self.listdids)
# Changing the replica statuses as follows:
# --------------------------------------------------------------------------------------------
# Name State(s) declared on MOCK_RECOVERY State(s) declared on MOCK_SUSPICIOUS
# --------------------------------------------------------------------------------------------
# tmp_file1 available suspicious (available)
# tmp_file2 available suspicious + bad (unavailable)
# tmp_file3 unavailable suspicious (available)
# --------------------------------------------------------------------------------------------
for replica in replicalist:
suspicious_pfns = replica['rses'][self.rse4suspicious_id]
for i in range(3):
print("Declaring suspicious file replica: " +
suspicious_pfns[0])
self.replica_client.declare_suspicious_file_replicas([
suspicious_pfns[0],
], 'This is a good reason.')
sleep(1)
if replica['name'] == path.basename(self.tmp_file2):
print("Declaring bad file replica: " + suspicious_pfns[0])
self.replica_client.declare_bad_file_replicas([
suspicious_pfns[0],
], 'This is a good reason')
if replica['name'] == path.basename(self.tmp_file3):
print("Updating replica state as unavailable: " +
replica['rses'][self.rse4recovery_id][0])
update_replica_state(self.rse4recovery_id, self.internal_scope,
path.basename(self.tmp_file3),
ReplicaState.UNAVAILABLE)
# Gather replica info after setting initial replica statuses
replicalist = list_replicas(dids=self.listdids)
# Checking if the status changes were effective
for replica in replicalist:
if replica['name'] == path.basename(self.tmp_file1):
assert replica['states'][self.rse4suspicious_id] == 'AVAILABLE'
assert replica['states'][self.rse4recovery_id] == 'AVAILABLE'
if replica['name'] == path.basename(self.tmp_file2):
assert (self.rse4suspicious_id in replica['states']) is False
assert replica['states'][self.rse4recovery_id] == 'AVAILABLE'
if replica['name'] == path.basename(self.tmp_file3):
assert replica['states'][self.rse4suspicious_id] == 'AVAILABLE'
assert (self.rse4recovery_id in replica['states']) is False
# Checking if only self.tmp_file2 is declared as 'BAD'
self.from_date = datetime.now() - timedelta(days=1)
bad_replicas_list = list_bad_replicas_status(
rse_id=self.rse4suspicious_id,
younger_than=self.from_date,
**self.vo)
bad_checklist = [(badf['name'], badf['rse_id'], badf['state'])
for badf in bad_replicas_list]
assert (path.basename(self.tmp_file2), self.rse4suspicious_id,
BadFilesStatus.BAD) in bad_checklist
assert (path.basename(self.tmp_file1), self.rse4suspicious_id,
BadFilesStatus.BAD) not in bad_checklist
assert (path.basename(self.tmp_file3), self.rse4suspicious_id,
BadFilesStatus.BAD) not in bad_checklist
bad_replicas_list = list_bad_replicas_status(
rse_id=self.rse4recovery_id,
younger_than=self.from_date,
**self.vo)
bad_checklist = [(badf['name'], badf['rse_id'], badf['state'])
for badf in bad_replicas_list]
assert (path.basename(self.tmp_file1), self.rse4recovery_id,
BadFilesStatus.BAD) not in bad_checklist
assert (path.basename(self.tmp_file2), self.rse4recovery_id,
BadFilesStatus.BAD) not in bad_checklist
assert (path.basename(self.tmp_file3), self.rse4recovery_id,
BadFilesStatus.BAD) not in bad_checklist
n_events = 0
for dbs_file in dbs_files:
if dbs_file['file'][0]['name'] == filename:
size = dbs_file['file'][0]['size']
adler32 = dbs_file['file'][0]['adler32']
n_events = dbs_file['file'][0]['nevents']
break
return size, adler32, n_events
if __name__ == '__main__':
aClient = AccountClient(account='ewv', auth_type='x509_proxy')
dClient = DIDClient(account='ewv', auth_type='x509_proxy')
rClient = ReplicaClient(account='ewv', auth_type='x509_proxy')
lClient = LifetimeClient(account='ewv', auth_type='x509_proxy')
ruleClient = RuleClient(account='ewv', auth_type='x509_proxy')
print("Connected to rucio as %s" % aClient.whoami()['account'])
# Make Rucio container and dataset to correspond to CMS dataset
try:
print("Adding dataset %s" % RUCIO_CONTAINER)
status = dClient.add_container(scope='cms', name=RUCIO_CONTAINER, lifetime=DAYS_TO_LIVE*24*3600)
print(' Status for add_container', status)
except DataIdentifierAlreadyExists:
print('Container already exists')
# try:
import collections
import re
import json
import logging
from rucio.client.rseclient import RSEClient
from rucio.client.didclient import DIDClient
from rucio.client.replicaclient import ReplicaClient
from kafka import KafkaProducer
from datetime import datetime
rucio_account = 'navila'
# DID Client
didclient = DIDClient(account=rucio_account)
# Replica Client
replicaclient = ReplicaClient(account=rucio_account)
# RSE Client
rseclient = RSEClient(account=rucio_account)
rselist = rseclient.list_rses()
# kafka scripts
logger = logging.getLogger(__name__)
logging.basicConfig(
level=logging.DEBUG,
format='%(asctime)s [%(levelname)s] %(name)s - %(message)s')
def vs(doc):
return json.dumps(doc).encode(encoding='utf-8', errors='replace')
#!/usr/bin/env python
from rucio.client.replicaclient import ReplicaClient
client = ReplicaClient()
#data = {
# 'pfn': "root://134.79.103.90:2076//psdm/rucio/rte/rte01/xtc/rte01-r0001-s01-c00.xtc",
# 'bytes': 104857600,
# 'md5': 'bfec14dd0fd2e733df4a7d00511f5a0c',
# 'name': "xtc.file.rte01-r0001-s01-c00.xtc",
# 'scope': "rte01",
#}
data = {
'pfn':
"root://134.79.103.90:2076//psdm/rucio/rte/rte01/xtc/rte01-r0002-s02-c00.xtc",
'bytes': 1073741824,
'adler32': '897ea592',
'md5': 'a0c84f769c76989f87a44b59409d73ea',
'name': "rte.rte01.xtc.rte01-r0002-s02-c00.xtc",
'scope': "psdm",
}
client.add_replicas('LCLS_DATA', [data])
