def stageIn(self, source, destination, fspec):
"""
Query HTTP for etag, then symlink to the pilot working directory.
:param source: original file location
:param destination: where to create the link
:param fspec: dictionary containing destination replicas, scope, lfn
:return: destination file details (checksumtype, checksum, size)
"""
self.log('source: %s' % str(source))
self.log('destination: %s' % str(destination))
self.log('fspec: %s' % str(fspec))
self.log('fspec.scope: %s' % str(fspec.scope))
self.log('fspec.lfn: %s' % str(fspec.lfn))
self.log('fspec.ddmendpoint: %s' % str(fspec.ddmendpoint))
# figure out the HTTP SURL from Rucio
from rucio.client import ReplicaClient
rc = ReplicaClient()
http_surl_reps = [
r for r in rc.list_replicas(dids=[{
'scope': fspec.scope,
'name': fspec.lfn
}],
schemes=['https'],
rse_expression=fspec.ddmendpoint)
]
self.log('http_surl_reps: %s' % http_surl_reps)
http_surl = http_surl_reps[0]['rses'][fspec.ddmendpoint][0].rsplit(
'_-')[0]
self.log('http_surl: %s' % http_surl)
# retrieve the TURL from the webdav etag
cmd = 'davix-http --capath /cvmfs/atlas.cern.ch/repo/ATLASLocalRootBase/etc/grid-security-emi/certificates --cert $X509_USER_PROXY -X PROPFIND %s' % http_surl
self.log('ETAG retrieval: %s' % cmd)
try:
timer = TimerCommand(cmd)
rcode, output = timer.run(timeout=10)
except Exception, e:
self.log('FATAL: could not retrieve STORM WebDAV ETag: %s' % e)
raise PilotException('Could not retrieve STORM WebDAV ETag: %s' %
e)
def stageIn(self, source, destination, fspec):
"""
Query HTTP for etag, then symlink to the pilot working directory.
:param source: original file location
:param destination: where to create the link
:param fspec: dictionary containing destination replicas, scope, lfn
:return: destination file details (checksumtype, checksum, size)
"""
self.log('source: %s' % str(source))
self.log('destination: %s' % str(destination))
self.log('fspec: %s' % str(fspec))
self.log('fspec.scope: %s' % str(fspec.scope))
self.log('fspec.lfn: %s' % str(fspec.lfn))
self.log('fspec.ddmendpoint: %s' % str(fspec.ddmendpoint))
# figure out the HTTP SURL from Rucio
from rucio.client import ReplicaClient
rc = ReplicaClient()
http_surl_reps = [r for r in rc.list_replicas(dids=[{'scope': fspec.scope,
'name': fspec.lfn}],
schemes=['davs'],
rse_expression=fspec.ddmendpoint)]
self.log('http_surl_reps: %s' % http_surl_reps)
http_surl = http_surl_reps[0]['rses'][fspec.ddmendpoint][0].rsplit('_-')[0]
self.log('http_surl: %s' % http_surl)
# retrieve the TURL from the webdav etag
cmd = 'davix-http --capath /cvmfs/atlas.cern.ch/repo/ATLASLocalRootBase/etc/grid-security-emi/certificates --cert $X509_USER_PROXY -X PROPFIND %s' % http_surl
self.log('ETAG retrieval: %s' % cmd)
try:
timer = TimerCommand(cmd)
rcode, output = timer.run(timeout=10)
except Exception, e:
self.log('FATAL: could not retrieve STORM WebDAV ETag: %s' % e)
raise PilotException('Could not retrieve STORM WebDAV ETag: %s' % e)
def setup(self):
self.rc = ReplicaClient()
self.client_location_without_proxy = {
'ip': '192.168.0.1',
'fqdn': 'anomalous-materials.blackmesa.com',
'site': 'BLACKMESA'
}
self.rse_without_proxy = rse_name_generator()
add_rse(self.rse_without_proxy)
add_rse_attribute(rse=self.rse_without_proxy,
key='site',
value='BLACKMESA')
self.client_location_with_proxy = {
'ip': '10.0.1.1',
'fqdn': 'test-chamber.aperture.com',
'site': 'APERTURE'
}
self.rse_with_proxy = rse_name_generator()
add_rse(self.rse_with_proxy)
add_rse_attribute(rse=self.rse_with_proxy,
key='root-proxy-internal',
value='root://proxy.aperture.com:1094')
add_rse_attribute(rse=self.rse_with_proxy,
key='site',
value='APERTURE')
self.files = [{
'scope': 'mock',
'name': 'half-life_%s' % i,
'bytes': 1234L,
'adler32': 'deadbeef',
'meta': {
'events': 666
}
} for i in xrange(1, 4)]
def test_replica_sorting(self):
""" REPLICA (CORE): Test the correct sorting of the replicas across WAN and LAN """
self.rc = ReplicaClient()
self.rse1 = 'APERTURE_%s' % rse_name_generator()
self.rse2 = 'BLACKMESA_%s' % rse_name_generator()
self.rse1_id = add_rse(self.rse1, **self.vo)
self.rse2_id = add_rse(self.rse2, **self.vo)
add_rse_attribute(rse_id=self.rse1_id, key='site', value='APERTURE')
add_rse_attribute(rse_id=self.rse2_id, key='site', value='BLACKMESA')
self.files = [{
'scope': InternalScope('mock', **self.vo),
'name': 'element_0',
'bytes': 1234,
'adler32': 'deadbeef'
}]
root = InternalAccount('root', **self.vo)
add_replicas(rse_id=self.rse1_id, files=self.files, account=root)
add_replicas(rse_id=self.rse2_id, files=self.files, account=root)
add_protocol(
self.rse1_id, {
'scheme': 'root',
'hostname': 'root.aperture.com',
'port': 1409,
'prefix': '//test/chamber/',
'impl': 'rucio.rse.protocols.xrootd.Default',
'domains': {
'lan': {
'read': 1,
'write': 1,
'delete': 1
},
'wan': {
'read': 1,
'write': 1,
'delete': 1
}
}
})
add_protocol(
self.rse1_id, {
'scheme': 'davs',
'hostname': 'davs.aperture.com',
'port': 443,
'prefix': '/test/chamber/',
'impl': 'rucio.rse.protocols.gfal.Default',
'domains': {
'lan': {
'read': 2,
'write': 2,
'delete': 2
},
'wan': {
'read': 2,
'write': 2,
'delete': 2
}
}
})
add_protocol(
self.rse1_id, {
'scheme': 'gsiftp',
'hostname': 'gsiftp.aperture.com',
'port': 8446,
'prefix': '/test/chamber/',
'impl': 'rucio.rse.protocols.gfal.Default',
'domains': {
'lan': {
'read': 0,
'write': 0,
'delete': 0
},
'wan': {
'read': 3,
'write': 3,
'delete': 3
}
}
})
add_protocol(
self.rse2_id, {
'scheme': 'gsiftp',
'hostname': 'gsiftp.blackmesa.com',
'port': 8446,
'prefix': '/lambda/complex/',
'impl': 'rucio.rse.protocols.gfal.Default',
'domains': {
'lan': {
'read': 2,
'write': 2,
'delete': 2
},
'wan': {
'read': 1,
'write': 1,
'delete': 1
}
}
})
add_protocol(
self.rse2_id, {
'scheme': 'davs',
'hostname': 'davs.blackmesa.com',
'port': 443,
'prefix': '/lambda/complex/',
'impl': 'rucio.rse.protocols.gfal.Default',
'domains': {
'lan': {
'read': 0,
'write': 0,
'delete': 0
},
'wan': {
'read': 2,
'write': 2,
'delete': 2
}
}
})
add_protocol(
self.rse2_id, {
'scheme': 'root',
'hostname': 'root.blackmesa.com',
'port': 1409,
'prefix': '//lambda/complex/',
'impl': 'rucio.rse.protocols.xrootd.Default',
'domains': {
'lan': {
'read': 1,
'write': 1,
'delete': 1
},
'wan': {
'read': 3,
'write': 3,
'delete': 3
}
}
})
replicas = [
r for r in self.rc.list_replicas(
dids=[{
'scope': 'mock',
'name': f['name'],
'type': 'FILE'
} for f in self.files],
schemes=['root', 'gsiftp', 'davs'],
client_location={'site': 'APERTURE'})
]
pfns = [r['pfns'] for r in replicas][0]
assert_equal(len(pfns.keys()), 5)
assert_equal(
pfns[
'root://root.aperture.com:1409//test/chamber/mock/58/b5/element_0']
['domain'], 'lan')
assert_equal(
pfns[
'root://root.aperture.com:1409//test/chamber/mock/58/b5/element_0']
['priority'], 1)
assert_equal(
pfns[
'davs://davs.aperture.com:443/test/chamber/mock/58/b5/element_0']
['domain'], 'lan')
assert_equal(
pfns[
'davs://davs.aperture.com:443/test/chamber/mock/58/b5/element_0']
['priority'], 2)
assert_equal(
pfns[
'gsiftp://gsiftp.blackmesa.com:8446/lambda/complex/mock/58/b5/element_0']
['domain'], 'wan')
assert_equal(
pfns[
'gsiftp://gsiftp.blackmesa.com:8446/lambda/complex/mock/58/b5/element_0']
['priority'], 3)
assert_equal(
pfns[
'davs://davs.blackmesa.com:443/lambda/complex/mock/58/b5/element_0']
['domain'], 'wan')
assert_equal(
pfns[
'davs://davs.blackmesa.com:443/lambda/complex/mock/58/b5/element_0']
['priority'], 4)
assert_equal(
pfns[
'root://root.blackmesa.com:1409//lambda/complex/mock/58/b5/element_0']
['domain'], 'wan')
assert_equal(
pfns[
'root://root.blackmesa.com:1409//lambda/complex/mock/58/b5/element_0']
['priority'], 5)
replicas = [
r for r in self.rc.list_replicas(
dids=[{
'scope': 'mock',
'name': f['name'],
'type': 'FILE'
} for f in self.files],
schemes=['root', 'gsiftp', 'davs'],
client_location={'site': 'BLACKMESA'})
]
pfns = [r['pfns'] for r in replicas][0]
assert_equal(len(pfns.keys()), 5)
assert_equal(
pfns[
'root://root.blackmesa.com:1409//lambda/complex/mock/58/b5/element_0']
['domain'], 'lan')
assert_equal(
pfns[
'root://root.blackmesa.com:1409//lambda/complex/mock/58/b5/element_0']
['priority'], 1)
assert_equal(
pfns[
'gsiftp://gsiftp.blackmesa.com:8446/lambda/complex/mock/58/b5/element_0']
['domain'], 'lan')
assert_equal(
pfns[
'gsiftp://gsiftp.blackmesa.com:8446/lambda/complex/mock/58/b5/element_0']
['priority'], 2)
assert_equal(
pfns[
'root://root.aperture.com:1409//test/chamber/mock/58/b5/element_0']
['domain'], 'wan')
assert_equal(
pfns[
'root://root.aperture.com:1409//test/chamber/mock/58/b5/element_0']
['priority'], 3)
assert_equal(
pfns[
'davs://davs.aperture.com:443/test/chamber/mock/58/b5/element_0']
['domain'], 'wan')
assert_equal(
pfns[
'davs://davs.aperture.com:443/test/chamber/mock/58/b5/element_0']
['priority'], 4)
assert_equal(
pfns[
'gsiftp://gsiftp.aperture.com:8446/test/chamber/mock/58/b5/element_0']
['domain'], 'wan')
assert_equal(
pfns[
'gsiftp://gsiftp.aperture.com:8446/test/chamber/mock/58/b5/element_0']
['priority'], 5)
replicas = [
r
for r in self.rc.list_replicas(dids=[{
'scope': 'mock',
'name': f['name'],
'type': 'FILE'
} for f in self.files],
schemes=['root', 'gsiftp', 'davs'],
client_location={'site': 'XEN'})
]
pfns = [r['pfns'] for r in replicas][0]
assert_equal(len(pfns.keys()), 6)
# TODO: intractable until RSE sorting is enabled
assert_equal(
pfns[
'gsiftp://gsiftp.blackmesa.com:8446/lambda/complex/mock/58/b5/element_0']
['domain'], 'wan')
assert_in(
pfns[
'gsiftp://gsiftp.blackmesa.com:8446/lambda/complex/mock/58/b5/element_0']
['priority'], [1, 2])
assert_equal(
pfns[
'root://root.aperture.com:1409//test/chamber/mock/58/b5/element_0']
['domain'], 'wan')
assert_in(
pfns[
'root://root.aperture.com:1409//test/chamber/mock/58/b5/element_0']
['priority'], [1, 2])
assert_equal(
pfns[
'davs://davs.aperture.com:443/test/chamber/mock/58/b5/element_0']
['domain'], 'wan')
assert_in(
pfns[
'davs://davs.aperture.com:443/test/chamber/mock/58/b5/element_0']
['priority'], [3, 4])
assert_equal(
pfns[
'davs://davs.blackmesa.com:443/lambda/complex/mock/58/b5/element_0']
['domain'], 'wan')
assert_in(
pfns[
'davs://davs.blackmesa.com:443/lambda/complex/mock/58/b5/element_0']
['priority'], [3, 4])
assert_equal(
pfns[
'gsiftp://gsiftp.aperture.com:8446/test/chamber/mock/58/b5/element_0']
['domain'], 'wan')
assert_in(
pfns[
'gsiftp://gsiftp.aperture.com:8446/test/chamber/mock/58/b5/element_0']
['priority'], [5, 6])
assert_equal(
pfns[
'root://root.blackmesa.com:1409//lambda/complex/mock/58/b5/element_0']
['domain'], 'wan')
assert_in(
pfns[
'root://root.blackmesa.com:1409//lambda/complex/mock/58/b5/element_0']
['priority'], [5, 6])
ml = self.rc.list_replicas(dids=[{
'scope': 'mock',
'name': f['name'],
'type': 'FILE'
} for f in self.files],
schemes=['root', 'gsiftp', 'davs'],
metalink=True,
client_location={'site': 'APERTURE'})
assert_in(
'domain="lan" priority="1" client_extract="false">root://root.aperture.com:1409//test/chamber/mock/58/b5/element_0',
ml)
assert_in(
'domain="lan" priority="2" client_extract="false">davs://davs.aperture.com:443/test/chamber/mock/58/b5/element_0',
ml)
assert_in(
'domain="wan" priority="3" client_extract="false">gsiftp://gsiftp.blackmesa.com:8446/lambda/complex/mock/58/b5/element_0',
ml)
assert_in(
'domain="wan" priority="4" client_extract="false">davs://davs.blackmesa.com:443/lambda/complex/mock/58/b5/element_0',
ml)
assert_in(
'domain="wan" priority="5" client_extract="false">root://root.blackmesa.com:1409//lambda/complex/mock/58/b5/element_0',
ml)
assert_not_in('priority="6"', ml)
ml = self.rc.list_replicas(dids=[{
'scope': 'mock',
'name': f['name'],
'type': 'FILE'
} for f in self.files],
schemes=['root', 'gsiftp', 'davs'],
metalink=True,
client_location={'site': 'BLACKMESA'})
assert_in(
'domain="lan" priority="1" client_extract="false">root://root.blackmesa.com:1409//lambda/complex/mock/58/b5/element_0',
ml)
assert_in(
'domain="lan" priority="2" client_extract="false">gsiftp://gsiftp.blackmesa.com:8446/lambda/complex/mock/58/b5/element_0',
ml)
assert_in(
'domain="wan" priority="3" client_extract="false">root://root.aperture.com:1409//test/chamber/mock/58/b5/element_0',
ml)
assert_in(
'domain="wan" priority="4" client_extract="false">davs://davs.aperture.com:443/test/chamber/mock/58/b5/element_0',
ml)
assert_in(
'domain="wan" priority="5" client_extract="false">gsiftp://gsiftp.aperture.com:8446/test/chamber/mock/58/b5/element_0',
ml)
assert_not_in('priority="6"', ml)
# TODO: intractable until RSE sorting is enabled
# ml = self.rc.list_replicas(dids=[{'scope': 'mock',
# 'name': f['name'],
# 'type': 'FILE'} for f in self.files],
# schemes=['root', 'gsiftp', 'davs'],
# metalink=True,
# client_location={'site': 'XEN'})
# assert_in('domain="wan" priority="1">root://root.aperture.com:1409//test/chamber/mock/58/b5/element_0', ml)
# assert_in('domain="wan" priority="2">gsiftp://gsiftp.blackmesa.com:8446/lambda/complex/mock/58/b5/element_0', ml)
# assert_in('domain="wan" priority="3">davs://davs.aperture.com:443/test/chamber/mock/58/b5/element_0', ml)
# assert_in('domain="wan" priority="4">davs://davs.blackmesa.com:443/lambda/complex/mock/58/b5/element_0', ml)
# assert_in('domain="wan" priority="5">gsiftp://gsiftp.aperture.com:8446/test/chamber/mock/58/b5/element_0', ml)
# assert_in('domain="wan" priority="6">root://root.blackmesa.com:1409//lambda/complex/mock/58/b5/element_0', ml)
# assert_not_in('priority="7"', ml)
# ensure correct handling of disabled protocols
add_protocol(
self.rse1_id, {
'scheme': 'root',
'hostname': 'root2.aperture.com',
'port': 1409,
'prefix': '//test/chamber/',
'impl': 'rucio.rse.protocols.xrootd.Default',
'domains': {
'lan': {
'read': 1,
'write': 1,
'delete': 1
},
'wan': {
'read': 0,
'write': 0,
'delete': 0
}
}
})
ml = self.rc.list_replicas(dids=[{
'scope': 'mock',
'name': f['name'],
'type': 'FILE'
} for f in self.files],
schemes=['root', 'gsiftp', 'davs'],
metalink=True,
client_location={'site': 'BLACKMESA'})
assert_in(
'domain="lan" priority="1" client_extract="false">root://root.blackmesa.com:1409//lambda/complex/mock/58/b5/element_0',
ml)
assert_in(
'domain="lan" priority="2" client_extract="false">gsiftp://gsiftp.blackmesa.com:8446/lambda/complex/mock/58/b5/element_0',
ml)
assert_in(
'domain="wan" priority="3" client_extract="false">root://root.aperture.com:1409//test/chamber/mock/58/b5/element_0',
ml)
assert_in(
'domain="wan" priority="4" client_extract="false">davs://davs.aperture.com:443/test/chamber/mock/58/b5/element_0',
ml)
assert_in(
'domain="wan" priority="5" client_extract="false">gsiftp://gsiftp.aperture.com:8446/test/chamber/mock/58/b5/element_0',
ml)
assert_not_in('priority="6"', ml)
delete_replicas(rse_id=self.rse1_id, files=self.files)
delete_replicas(rse_id=self.rse2_id, files=self.files)
del_rse(self.rse1_id)
del_rse(self.rse2_id)
def setup(self):
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
self.vo = {'vo': 'tst'}
else:
self.vo = {}
self.rc = ReplicaClient()
self.client_location_without_proxy = {'ip': '192.168.0.1',
'fqdn': 'anomalous-materials.blackmesa.com',
'site': 'BLACKMESA1'}
self.rse_without_proxy = rse_name_generator()
self.rse_without_proxy_id = add_rse(self.rse_without_proxy, **self.vo)
add_rse_attribute(rse_id=self.rse_without_proxy_id,
key='site',
value='BLACKMESA1')
self.client_location_with_proxy = {'ip': '10.0.1.1',
'fqdn': 'test-chamber.aperture.com',
'site': 'APERTURE1'}
self.rse_with_proxy = rse_name_generator()
self.rse_with_proxy_id = add_rse(self.rse_with_proxy, **self.vo)
add_rse_attribute(rse_id=self.rse_with_proxy_id,
key='site',
value='APERTURE1')
# APERTURE1 site has an internal proxy
config_set('root-proxy-internal', 'APERTURE1', 'proxy.aperture.com:1094')
self.files = [{'scope': InternalScope('mock', **self.vo),
'name': 'half-life_%s' % i,
'bytes': 1234,
'adler32': 'deadbeef',
'meta': {'events': 666}} for i in range(1, 4)]
for rse_id in [self.rse_with_proxy_id, self.rse_without_proxy_id]:
add_replicas(rse_id=rse_id,
files=self.files,
account=InternalAccount('root', **self.vo),
ignore_availability=True)
add_protocol(self.rse_without_proxy_id, {'scheme': 'root',
'hostname': 'root.blackmesa.com',
'port': 1409,
'prefix': '//training/facility/',
'impl': 'rucio.rse.protocols.xrootd.Default',
'domains': {
'lan': {'read': 1,
'write': 1,
'delete': 1},
'wan': {'read': 1,
'write': 1,
'delete': 1}}})
add_protocol(self.rse_with_proxy_id, {'scheme': 'root',
'hostname': 'root.aperture.com',
'port': 1409,
'prefix': '//test/chamber/',
'impl': 'rucio.rse.protocols.xrootd.Default',
'domains': {
'lan': {'read': 1,
'write': 1,
'delete': 1},
'wan': {'read': 1,
'write': 1,
'delete': 1}}})
class TestROOTProxy(object):
def setup(self):
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
self.vo = {'vo': 'tst'}
else:
self.vo = {}
self.rc = ReplicaClient()
self.client_location_without_proxy = {'ip': '192.168.0.1',
'fqdn': 'anomalous-materials.blackmesa.com',
'site': 'BLACKMESA1'}
self.rse_without_proxy = rse_name_generator()
self.rse_without_proxy_id = add_rse(self.rse_without_proxy, **self.vo)
add_rse_attribute(rse_id=self.rse_without_proxy_id,
key='site',
value='BLACKMESA1')
self.client_location_with_proxy = {'ip': '10.0.1.1',
'fqdn': 'test-chamber.aperture.com',
'site': 'APERTURE1'}
self.rse_with_proxy = rse_name_generator()
self.rse_with_proxy_id = add_rse(self.rse_with_proxy, **self.vo)
add_rse_attribute(rse_id=self.rse_with_proxy_id,
key='site',
value='APERTURE1')
# APERTURE1 site has an internal proxy
config_set('root-proxy-internal', 'APERTURE1', 'proxy.aperture.com:1094')
self.files = [{'scope': InternalScope('mock', **self.vo),
'name': 'half-life_%s' % i,
'bytes': 1234,
'adler32': 'deadbeef',
'meta': {'events': 666}} for i in range(1, 4)]
for rse_id in [self.rse_with_proxy_id, self.rse_without_proxy_id]:
add_replicas(rse_id=rse_id,
files=self.files,
account=InternalAccount('root', **self.vo),
ignore_availability=True)
add_protocol(self.rse_without_proxy_id, {'scheme': 'root',
'hostname': 'root.blackmesa.com',
'port': 1409,
'prefix': '//training/facility/',
'impl': 'rucio.rse.protocols.xrootd.Default',
'domains': {
'lan': {'read': 1,
'write': 1,
'delete': 1},
'wan': {'read': 1,
'write': 1,
'delete': 1}}})
add_protocol(self.rse_with_proxy_id, {'scheme': 'root',
'hostname': 'root.aperture.com',
'port': 1409,
'prefix': '//test/chamber/',
'impl': 'rucio.rse.protocols.xrootd.Default',
'domains': {
'lan': {'read': 1,
'write': 1,
'delete': 1},
'wan': {'read': 1,
'write': 1,
'delete': 1}}})
def tearDown(self):
for rse_id in [self.rse_with_proxy_id, self.rse_without_proxy_id]:
delete_replicas(rse_id=rse_id, files=self.files)
del_rse(self.rse_with_proxy_id)
del_rse(self.rse_without_proxy_id)
def test_client_list_replicas(self):
""" ROOT (CLIENT): Test internal proxy prepend """
# no proxy involved
replicas = [r for r in self.rc.list_replicas(dids=[{'scope': 'mock',
'name': f['name'],
'type': 'FILE'} for f in self.files],
rse_expression=self.rse_without_proxy,
client_location=self.client_location_without_proxy)]
expected_pfns = ['root://root.blackmesa.com:1409//training/facility/mock/c9/df/half-life_1',
'root://root.blackmesa.com:1409//training/facility/mock/c1/8d/half-life_2',
'root://root.blackmesa.com:1409//training/facility/mock/16/30/half-life_3']
found_pfns = [replica['pfns'].keys()[0] for replica in replicas]
assert_equal(sorted(found_pfns), sorted(expected_pfns))
# outgoing proxy needs to be prepended
replicas = [r for r in self.rc.list_replicas(dids=[{'scope': 'mock',
'name': f['name'],
'type': 'FILE'} for f in self.files],
rse_expression=self.rse_without_proxy,
client_location=self.client_location_with_proxy)]
expected_pfns = ['root://proxy.aperture.com:1094//root://root.blackmesa.com:1409//training/facility/mock/c9/df/half-life_1',
'root://proxy.aperture.com:1094//root://root.blackmesa.com:1409//training/facility/mock/c1/8d/half-life_2',
'root://proxy.aperture.com:1094//root://root.blackmesa.com:1409//training/facility/mock/16/30/half-life_3']
found_pfns = [replica['pfns'].keys()[0] for replica in replicas]
assert_equal(sorted(found_pfns), sorted(expected_pfns))
# outgoing proxy at destination does not matter
replicas = [r for r in self.rc.list_replicas(dids=[{'scope': 'mock',
'name': f['name'],
'type': 'FILE'} for f in self.files],
rse_expression=self.rse_with_proxy,
client_location=self.client_location_without_proxy)]
expected_pfns = ['root://root.aperture.com:1409//test/chamber/mock/c9/df/half-life_1',
'root://root.aperture.com:1409//test/chamber/mock/c1/8d/half-life_2',
'root://root.aperture.com:1409//test/chamber/mock/16/30/half-life_3']
found_pfns = [replica['pfns'].keys()[0] for replica in replicas]
assert_equal(sorted(found_pfns), sorted(expected_pfns))
# outgoing proxy does not matter when staying at site
replicas = [r for r in self.rc.list_replicas(dids=[{'scope': 'mock',
'name': f['name'],
'type': 'FILE'} for f in self.files],
rse_expression=self.rse_with_proxy,
client_location=self.client_location_with_proxy)]
expected_pfns = ['root://root.aperture.com:1409//test/chamber/mock/c9/df/half-life_1',
'root://root.aperture.com:1409//test/chamber/mock/c1/8d/half-life_2',
'root://root.aperture.com:1409//test/chamber/mock/16/30/half-life_3']
found_pfns = [replica['pfns'].keys()[0] for replica in replicas]
assert_equal(sorted(found_pfns), sorted(expected_pfns))
def test_redirect_metalink_list_replicas(self):
""" ROOT (REDIRECT REST): Test internal proxy prepend with metalink"""
mw = []
# default behaviour - no location -> no proxy
res = TestApp(redirect_app.wsgifunc(*mw)).get('/mock/half-life_1/metalink', expect_errors=True)
assert_in('root://root.blackmesa.com:1409//training/facility/mock/c9/df/half-life_1', res.body)
assert_in('root://root.aperture.com:1409//test/chamber/mock/c9/df/half-life_1', res.body)
assert_not_in('proxy', res.body)
res = TestApp(redirect_app.wsgifunc(*mw)).get('/mock/half-life_2/metalink', expect_errors=True)
assert_in('root://root.blackmesa.com:1409//training/facility/mock/c1/8d/half-life_2', res.body)
assert_in('root://root.aperture.com:1409//test/chamber/mock/c1/8d/half-life_2', res.body)
assert_not_in('proxy', res.body)
res = TestApp(redirect_app.wsgifunc(*mw)).get('/mock/half-life_3/metalink', expect_errors=True)
assert_in('root://root.blackmesa.com:1409//training/facility/mock/16/30/half-life_3', res.body)
assert_in('root://root.aperture.com:1409//test/chamber/mock/16/30/half-life_3', res.body)
assert_not_in('proxy', res.body)
# site without proxy
res = TestApp(redirect_app.wsgifunc(*mw)).get('/mock/half-life_1/metalink?%s' % urlencode(self.client_location_without_proxy),
expect_errors=True)
assert_in('root://root.blackmesa.com:1409//training/facility/mock/c9/df/half-life_1', res.body)
assert_in('root://root.aperture.com:1409//test/chamber/mock/c9/df/half-life_1', res.body)
assert_not_in('proxy', res.body)
res = TestApp(redirect_app.wsgifunc(*mw)).get('/mock/half-life_2/metalink?%s' % urlencode(self.client_location_without_proxy),
expect_errors=True)
assert_in('root://root.blackmesa.com:1409//training/facility/mock/c1/8d/half-life_2', res.body)
assert_in('root://root.aperture.com:1409//test/chamber/mock/c1/8d/half-life_2', res.body)
assert_not_in('proxy', res.body)
res = TestApp(redirect_app.wsgifunc(*mw)).get('/mock/half-life_3/metalink?%s' % urlencode(self.client_location_without_proxy),
expect_errors=True)
assert_in('root://root.blackmesa.com:1409//training/facility/mock/16/30/half-life_3', res.body)
assert_in('root://root.aperture.com:1409//test/chamber/mock/16/30/half-life_3', res.body)
assert_not_in('proxy', res.body)
# at location with outgoing proxy, prepend for wan replica
res = TestApp(redirect_app.wsgifunc(*mw)).get('/mock/half-life_1/metalink?%s' % urlencode(self.client_location_with_proxy),
expect_errors=True)
assert_in('root://proxy.aperture.com:1094//root://root.blackmesa.com:1409//training/facility/mock/c9/df/half-life_1', res.body)
assert_in('root://root.aperture.com:1409//test/chamber/mock/c9/df/half-life_1', res.body)
res = TestApp(redirect_app.wsgifunc(*mw)).get('/mock/half-life_2/metalink?%s' % urlencode(self.client_location_with_proxy),
expect_errors=True)
assert_in('root://proxy.aperture.com:1094//root://root.blackmesa.com:1409//training/facility/mock/c1/8d/half-life_2', res.body)
assert_in('root://root.aperture.com:1409//test/chamber/mock/c1/8d/half-life_2', res.body)
res = TestApp(redirect_app.wsgifunc(*mw)).get('/mock/half-life_3/metalink?%s' % urlencode(self.client_location_with_proxy),
expect_errors=True)
assert_in('root://proxy.aperture.com:1094//root://root.blackmesa.com:1409//training/facility/mock/16/30/half-life_3', res.body)
assert_in('root://root.aperture.com:1409//test/chamber/mock/16/30/half-life_3', res.body)
},
# {'wildcard': {'filename': 'EVNT*'}},
# {'wildcard': {'event': 'get_sm*'}},
{
'term': {
'event': 'get_sm'
}
},
# {'term': {'event': 'get_sm_a'}},
# {'term': {'event': 'download'}},
]
}
}
}
rc = ReplicaClient()
scroll = scan(client=es, index="traces", query=my_query)
count = 0
requests = []
for res in scroll:
r = res['_source']
# print(r)
scope = r['scope']
filen = r['filename']
files = r['filesize']
times = r['time_start'] * 1000
gen = rc.list_replicas(dids=[{
'scope': scope,
'name': filen
}],
},
# {'wildcard': {'filename': 'EVNT*'}},
# {'wildcard': {'event': 'get_sm*'}},
{
'term': {
'event': 'get_sm'
}
},
# {'term': {'event': 'get_sm_a'}},
# {'term': {'event': 'download'}},
]
}
}
}
rc = ReplicaClient()
scroll = scan(client=es, index="traces", query=my_query)
count = 0
requests = []
for res in scroll:
r = res['_source']
# print(r)
scope = r['scope']
filen = r['filename']
files = r['filesize']
times = r['time_start'] * 1000
gen = rc.list_replicas(dids=[{
'scope': scope,
'name': filen
}],
def setup(self):
self.rc = ReplicaClient()
self.client_location_without_proxy = {
'ip': '192.168.0.1',
'fqdn': 'anomalous-materials.blackmesa.com',
'site': 'BLACKMESA1'
}
self.rse_without_proxy = rse_name_generator()
add_rse(self.rse_without_proxy)
add_rse_attribute(rse=self.rse_without_proxy,
key='site',
value='BLACKMESA1')
self.client_location_with_proxy = {
'ip': '10.0.1.1',
'fqdn': 'test-chamber.aperture.com',
'site': 'APERTURE1'
}
self.rse_with_proxy = rse_name_generator()
add_rse(self.rse_with_proxy)
add_rse_attribute(rse=self.rse_with_proxy,
key='root-proxy-internal',
value='root://proxy.aperture.com:1094')
add_rse_attribute(rse=self.rse_with_proxy,
key='site',
value='APERTURE1')
self.files = [{
'scope': 'mock',
'name': 'half-life_%s' % i,
'bytes': 1234,
'adler32': 'deadbeef',
'meta': {
'events': 666
}
} for i in range(1, 4)]
for rse in [self.rse_with_proxy, self.rse_without_proxy]:
add_replicas(rse=rse,
files=self.files,
account='root',
ignore_availability=True)
add_protocol(
self.rse_without_proxy, {
'scheme': 'root',
'hostname': 'root.blackmesa.com',
'port': 1409,
'prefix': '//training/facility/',
'impl': 'rucio.rse.protocols.xrootd.Default',
'domains': {
'lan': {
'read': 1,
'write': 1,
'delete': 1
},
'wan': {
'read': 1,
'write': 1,
'delete': 1
}
}
})
add_protocol(
self.rse_with_proxy, {
'scheme': 'root',
'hostname': 'root.aperture.com',
'port': 1409,
'prefix': '//test/chamber/',
'impl': 'rucio.rse.protocols.xrootd.Default',
'domains': {
'lan': {
'read': 1,
'write': 1,
'delete': 1
},
'wan': {
'read': 1,
'write': 1,
'delete': 1
}
}
})
except pika.exceptions.AMQPConnectionError as eek:
rabbitmq = None
retry_count += 1
if retry_count < retries:
print(
"Failed to connect to RabbitMQ. Waiting before trying again"
)
time.sleep(retry_interval)
else:
print("Failed to connect to RabbitMQ. Giving Up")
raise eek
# Main Script
args = parser.parse_args()
site = args.site
prefix = args.prefix
threads = args.threads
print("--------- ServiceX DID Finder ----------------")
print("Threads: " + str(threads))
print("Site: " + str(site))
print("Prefix: " + str(prefix))
did_client = DIDClient()
replica_client = ReplicaClient()
rucio_adapter = RucioAdapter(did_client, replica_client)
init_rabbit_mq(args.rabbit_uri, retries=12, retry_interval=10)
