def stageIn(self, source, destination, fspec): """ Query HTTP for etag, then symlink to the pilot working directory. :param source: original file location :param destination: where to create the link :param fspec: dictionary containing destination replicas, scope, lfn :return: destination file details (checksumtype, checksum, size) """ self.log('source: %s' % str(source)) self.log('destination: %s' % str(destination)) self.log('fspec: %s' % str(fspec)) self.log('fspec.scope: %s' % str(fspec.scope)) self.log('fspec.lfn: %s' % str(fspec.lfn)) self.log('fspec.ddmendpoint: %s' % str(fspec.ddmendpoint)) # figure out the HTTP SURL from Rucio from rucio.client import ReplicaClient rc = ReplicaClient() http_surl_reps = [ r for r in rc.list_replicas(dids=[{ 'scope': fspec.scope, 'name': fspec.lfn }], schemes=['https'], rse_expression=fspec.ddmendpoint) ] self.log('http_surl_reps: %s' % http_surl_reps) http_surl = http_surl_reps[0]['rses'][fspec.ddmendpoint][0].rsplit( '_-')[0] self.log('http_surl: %s' % http_surl) # retrieve the TURL from the webdav etag cmd = 'davix-http --capath /cvmfs/atlas.cern.ch/repo/ATLASLocalRootBase/etc/grid-security-emi/certificates --cert $X509_USER_PROXY -X PROPFIND %s' % http_surl self.log('ETAG retrieval: %s' % cmd) try: timer = TimerCommand(cmd) rcode, output = timer.run(timeout=10) except Exception, e: self.log('FATAL: could not retrieve STORM WebDAV ETag: %s' % e) raise PilotException('Could not retrieve STORM WebDAV ETag: %s' % e)
def stageIn(self, source, destination, fspec): """ Query HTTP for etag, then symlink to the pilot working directory. :param source: original file location :param destination: where to create the link :param fspec: dictionary containing destination replicas, scope, lfn :return: destination file details (checksumtype, checksum, size) """ self.log('source: %s' % str(source)) self.log('destination: %s' % str(destination)) self.log('fspec: %s' % str(fspec)) self.log('fspec.scope: %s' % str(fspec.scope)) self.log('fspec.lfn: %s' % str(fspec.lfn)) self.log('fspec.ddmendpoint: %s' % str(fspec.ddmendpoint)) # figure out the HTTP SURL from Rucio from rucio.client import ReplicaClient rc = ReplicaClient() http_surl_reps = [r for r in rc.list_replicas(dids=[{'scope': fspec.scope, 'name': fspec.lfn}], schemes=['davs'], rse_expression=fspec.ddmendpoint)] self.log('http_surl_reps: %s' % http_surl_reps) http_surl = http_surl_reps[0]['rses'][fspec.ddmendpoint][0].rsplit('_-')[0] self.log('http_surl: %s' % http_surl) # retrieve the TURL from the webdav etag cmd = 'davix-http --capath /cvmfs/atlas.cern.ch/repo/ATLASLocalRootBase/etc/grid-security-emi/certificates --cert $X509_USER_PROXY -X PROPFIND %s' % http_surl self.log('ETAG retrieval: %s' % cmd) try: timer = TimerCommand(cmd) rcode, output = timer.run(timeout=10) except Exception, e: self.log('FATAL: could not retrieve STORM WebDAV ETag: %s' % e) raise PilotException('Could not retrieve STORM WebDAV ETag: %s' % e)
def setup(self): self.rc = ReplicaClient() self.client_location_without_proxy = { 'ip': '192.168.0.1', 'fqdn': 'anomalous-materials.blackmesa.com', 'site': 'BLACKMESA' } self.rse_without_proxy = rse_name_generator() add_rse(self.rse_without_proxy) add_rse_attribute(rse=self.rse_without_proxy, key='site', value='BLACKMESA') self.client_location_with_proxy = { 'ip': '10.0.1.1', 'fqdn': 'test-chamber.aperture.com', 'site': 'APERTURE' } self.rse_with_proxy = rse_name_generator() add_rse(self.rse_with_proxy) add_rse_attribute(rse=self.rse_with_proxy, key='root-proxy-internal', value='root://proxy.aperture.com:1094') add_rse_attribute(rse=self.rse_with_proxy, key='site', value='APERTURE') self.files = [{ 'scope': 'mock', 'name': 'half-life_%s' % i, 'bytes': 1234L, 'adler32': 'deadbeef', 'meta': { 'events': 666 } } for i in xrange(1, 4)]
def test_replica_sorting(self): """ REPLICA (CORE): Test the correct sorting of the replicas across WAN and LAN """ self.rc = ReplicaClient() self.rse1 = 'APERTURE_%s' % rse_name_generator() self.rse2 = 'BLACKMESA_%s' % rse_name_generator() self.rse1_id = add_rse(self.rse1, **self.vo) self.rse2_id = add_rse(self.rse2, **self.vo) add_rse_attribute(rse_id=self.rse1_id, key='site', value='APERTURE') add_rse_attribute(rse_id=self.rse2_id, key='site', value='BLACKMESA') self.files = [{ 'scope': InternalScope('mock', **self.vo), 'name': 'element_0', 'bytes': 1234, 'adler32': 'deadbeef' }] root = InternalAccount('root', **self.vo) add_replicas(rse_id=self.rse1_id, files=self.files, account=root) add_replicas(rse_id=self.rse2_id, files=self.files, account=root) add_protocol( self.rse1_id, { 'scheme': 'root', 'hostname': 'root.aperture.com', 'port': 1409, 'prefix': '//test/chamber/', 'impl': 'rucio.rse.protocols.xrootd.Default', 'domains': { 'lan': { 'read': 1, 'write': 1, 'delete': 1 }, 'wan': { 'read': 1, 'write': 1, 'delete': 1 } } }) add_protocol( self.rse1_id, { 'scheme': 'davs', 'hostname': 'davs.aperture.com', 'port': 443, 'prefix': '/test/chamber/', 'impl': 'rucio.rse.protocols.gfal.Default', 'domains': { 'lan': { 'read': 2, 'write': 2, 'delete': 2 }, 'wan': { 'read': 2, 'write': 2, 'delete': 2 } } }) add_protocol( self.rse1_id, { 'scheme': 'gsiftp', 'hostname': 'gsiftp.aperture.com', 'port': 8446, 'prefix': '/test/chamber/', 'impl': 'rucio.rse.protocols.gfal.Default', 'domains': { 'lan': { 'read': 0, 'write': 0, 'delete': 0 }, 'wan': { 'read': 3, 'write': 3, 'delete': 3 } } }) add_protocol( self.rse2_id, { 'scheme': 'gsiftp', 'hostname': 'gsiftp.blackmesa.com', 'port': 8446, 'prefix': '/lambda/complex/', 'impl': 'rucio.rse.protocols.gfal.Default', 'domains': { 'lan': { 'read': 2, 'write': 2, 'delete': 2 }, 'wan': { 'read': 1, 'write': 1, 'delete': 1 } } }) add_protocol( self.rse2_id, { 'scheme': 'davs', 'hostname': 'davs.blackmesa.com', 'port': 443, 'prefix': '/lambda/complex/', 'impl': 'rucio.rse.protocols.gfal.Default', 'domains': { 'lan': { 'read': 0, 'write': 0, 'delete': 0 }, 'wan': { 'read': 2, 'write': 2, 'delete': 2 } } }) add_protocol( self.rse2_id, { 'scheme': 'root', 'hostname': 'root.blackmesa.com', 'port': 1409, 'prefix': '//lambda/complex/', 'impl': 'rucio.rse.protocols.xrootd.Default', 'domains': { 'lan': { 'read': 1, 'write': 1, 'delete': 1 }, 'wan': { 'read': 3, 'write': 3, 'delete': 3 } } }) replicas = [ r for r in self.rc.list_replicas( dids=[{ 'scope': 'mock', 'name': f['name'], 'type': 'FILE' } for f in self.files], schemes=['root', 'gsiftp', 'davs'], client_location={'site': 'APERTURE'}) ] pfns = [r['pfns'] for r in replicas][0] assert_equal(len(pfns.keys()), 5) assert_equal( pfns[ 'root://root.aperture.com:1409//test/chamber/mock/58/b5/element_0'] ['domain'], 'lan') assert_equal( pfns[ 'root://root.aperture.com:1409//test/chamber/mock/58/b5/element_0'] ['priority'], 1) assert_equal( pfns[ 'davs://davs.aperture.com:443/test/chamber/mock/58/b5/element_0'] ['domain'], 'lan') assert_equal( pfns[ 'davs://davs.aperture.com:443/test/chamber/mock/58/b5/element_0'] ['priority'], 2) assert_equal( pfns[ 'gsiftp://gsiftp.blackmesa.com:8446/lambda/complex/mock/58/b5/element_0'] ['domain'], 'wan') assert_equal( pfns[ 'gsiftp://gsiftp.blackmesa.com:8446/lambda/complex/mock/58/b5/element_0'] ['priority'], 3) assert_equal( pfns[ 'davs://davs.blackmesa.com:443/lambda/complex/mock/58/b5/element_0'] ['domain'], 'wan') assert_equal( pfns[ 'davs://davs.blackmesa.com:443/lambda/complex/mock/58/b5/element_0'] ['priority'], 4) assert_equal( pfns[ 'root://root.blackmesa.com:1409//lambda/complex/mock/58/b5/element_0'] ['domain'], 'wan') assert_equal( pfns[ 'root://root.blackmesa.com:1409//lambda/complex/mock/58/b5/element_0'] ['priority'], 5) replicas = [ r for r in self.rc.list_replicas( dids=[{ 'scope': 'mock', 'name': f['name'], 'type': 'FILE' } for f in self.files], schemes=['root', 'gsiftp', 'davs'], client_location={'site': 'BLACKMESA'}) ] pfns = [r['pfns'] for r in replicas][0] assert_equal(len(pfns.keys()), 5) assert_equal( pfns[ 'root://root.blackmesa.com:1409//lambda/complex/mock/58/b5/element_0'] ['domain'], 'lan') assert_equal( pfns[ 'root://root.blackmesa.com:1409//lambda/complex/mock/58/b5/element_0'] ['priority'], 1) assert_equal( pfns[ 'gsiftp://gsiftp.blackmesa.com:8446/lambda/complex/mock/58/b5/element_0'] ['domain'], 'lan') assert_equal( pfns[ 'gsiftp://gsiftp.blackmesa.com:8446/lambda/complex/mock/58/b5/element_0'] ['priority'], 2) assert_equal( pfns[ 'root://root.aperture.com:1409//test/chamber/mock/58/b5/element_0'] ['domain'], 'wan') assert_equal( pfns[ 'root://root.aperture.com:1409//test/chamber/mock/58/b5/element_0'] ['priority'], 3) assert_equal( pfns[ 'davs://davs.aperture.com:443/test/chamber/mock/58/b5/element_0'] ['domain'], 'wan') assert_equal( pfns[ 'davs://davs.aperture.com:443/test/chamber/mock/58/b5/element_0'] ['priority'], 4) assert_equal( pfns[ 'gsiftp://gsiftp.aperture.com:8446/test/chamber/mock/58/b5/element_0'] ['domain'], 'wan') assert_equal( pfns[ 'gsiftp://gsiftp.aperture.com:8446/test/chamber/mock/58/b5/element_0'] ['priority'], 5) replicas = [ r for r in self.rc.list_replicas(dids=[{ 'scope': 'mock', 'name': f['name'], 'type': 'FILE' } for f in self.files], schemes=['root', 'gsiftp', 'davs'], client_location={'site': 'XEN'}) ] pfns = [r['pfns'] for r in replicas][0] assert_equal(len(pfns.keys()), 6) # TODO: intractable until RSE sorting is enabled assert_equal( pfns[ 'gsiftp://gsiftp.blackmesa.com:8446/lambda/complex/mock/58/b5/element_0'] ['domain'], 'wan') assert_in( pfns[ 'gsiftp://gsiftp.blackmesa.com:8446/lambda/complex/mock/58/b5/element_0'] ['priority'], [1, 2]) assert_equal( pfns[ 'root://root.aperture.com:1409//test/chamber/mock/58/b5/element_0'] ['domain'], 'wan') assert_in( pfns[ 'root://root.aperture.com:1409//test/chamber/mock/58/b5/element_0'] ['priority'], [1, 2]) assert_equal( pfns[ 'davs://davs.aperture.com:443/test/chamber/mock/58/b5/element_0'] ['domain'], 'wan') assert_in( pfns[ 'davs://davs.aperture.com:443/test/chamber/mock/58/b5/element_0'] ['priority'], [3, 4]) assert_equal( pfns[ 'davs://davs.blackmesa.com:443/lambda/complex/mock/58/b5/element_0'] ['domain'], 'wan') assert_in( pfns[ 'davs://davs.blackmesa.com:443/lambda/complex/mock/58/b5/element_0'] ['priority'], [3, 4]) assert_equal( pfns[ 'gsiftp://gsiftp.aperture.com:8446/test/chamber/mock/58/b5/element_0'] ['domain'], 'wan') assert_in( pfns[ 'gsiftp://gsiftp.aperture.com:8446/test/chamber/mock/58/b5/element_0'] ['priority'], [5, 6]) assert_equal( pfns[ 'root://root.blackmesa.com:1409//lambda/complex/mock/58/b5/element_0'] ['domain'], 'wan') assert_in( pfns[ 'root://root.blackmesa.com:1409//lambda/complex/mock/58/b5/element_0'] ['priority'], [5, 6]) ml = self.rc.list_replicas(dids=[{ 'scope': 'mock', 'name': f['name'], 'type': 'FILE' } for f in self.files], schemes=['root', 'gsiftp', 'davs'], metalink=True, client_location={'site': 'APERTURE'}) assert_in( 'domain="lan" priority="1" client_extract="false">root://root.aperture.com:1409//test/chamber/mock/58/b5/element_0', ml) assert_in( 'domain="lan" priority="2" client_extract="false">davs://davs.aperture.com:443/test/chamber/mock/58/b5/element_0', ml) assert_in( 'domain="wan" priority="3" client_extract="false">gsiftp://gsiftp.blackmesa.com:8446/lambda/complex/mock/58/b5/element_0', ml) assert_in( 'domain="wan" priority="4" client_extract="false">davs://davs.blackmesa.com:443/lambda/complex/mock/58/b5/element_0', ml) assert_in( 'domain="wan" priority="5" client_extract="false">root://root.blackmesa.com:1409//lambda/complex/mock/58/b5/element_0', ml) assert_not_in('priority="6"', ml) ml = self.rc.list_replicas(dids=[{ 'scope': 'mock', 'name': f['name'], 'type': 'FILE' } for f in self.files], schemes=['root', 'gsiftp', 'davs'], metalink=True, client_location={'site': 'BLACKMESA'}) assert_in( 'domain="lan" priority="1" client_extract="false">root://root.blackmesa.com:1409//lambda/complex/mock/58/b5/element_0', ml) assert_in( 'domain="lan" priority="2" client_extract="false">gsiftp://gsiftp.blackmesa.com:8446/lambda/complex/mock/58/b5/element_0', ml) assert_in( 'domain="wan" priority="3" client_extract="false">root://root.aperture.com:1409//test/chamber/mock/58/b5/element_0', ml) assert_in( 'domain="wan" priority="4" client_extract="false">davs://davs.aperture.com:443/test/chamber/mock/58/b5/element_0', ml) assert_in( 'domain="wan" priority="5" client_extract="false">gsiftp://gsiftp.aperture.com:8446/test/chamber/mock/58/b5/element_0', ml) assert_not_in('priority="6"', ml) # TODO: intractable until RSE sorting is enabled # ml = self.rc.list_replicas(dids=[{'scope': 'mock', # 'name': f['name'], # 'type': 'FILE'} for f in self.files], # schemes=['root', 'gsiftp', 'davs'], # metalink=True, # client_location={'site': 'XEN'}) # assert_in('domain="wan" priority="1">root://root.aperture.com:1409//test/chamber/mock/58/b5/element_0', ml) # assert_in('domain="wan" priority="2">gsiftp://gsiftp.blackmesa.com:8446/lambda/complex/mock/58/b5/element_0', ml) # assert_in('domain="wan" priority="3">davs://davs.aperture.com:443/test/chamber/mock/58/b5/element_0', ml) # assert_in('domain="wan" priority="4">davs://davs.blackmesa.com:443/lambda/complex/mock/58/b5/element_0', ml) # assert_in('domain="wan" priority="5">gsiftp://gsiftp.aperture.com:8446/test/chamber/mock/58/b5/element_0', ml) # assert_in('domain="wan" priority="6">root://root.blackmesa.com:1409//lambda/complex/mock/58/b5/element_0', ml) # assert_not_in('priority="7"', ml) # ensure correct handling of disabled protocols add_protocol( self.rse1_id, { 'scheme': 'root', 'hostname': 'root2.aperture.com', 'port': 1409, 'prefix': '//test/chamber/', 'impl': 'rucio.rse.protocols.xrootd.Default', 'domains': { 'lan': { 'read': 1, 'write': 1, 'delete': 1 }, 'wan': { 'read': 0, 'write': 0, 'delete': 0 } } }) ml = self.rc.list_replicas(dids=[{ 'scope': 'mock', 'name': f['name'], 'type': 'FILE' } for f in self.files], schemes=['root', 'gsiftp', 'davs'], metalink=True, client_location={'site': 'BLACKMESA'}) assert_in( 'domain="lan" priority="1" client_extract="false">root://root.blackmesa.com:1409//lambda/complex/mock/58/b5/element_0', ml) assert_in( 'domain="lan" priority="2" client_extract="false">gsiftp://gsiftp.blackmesa.com:8446/lambda/complex/mock/58/b5/element_0', ml) assert_in( 'domain="wan" priority="3" client_extract="false">root://root.aperture.com:1409//test/chamber/mock/58/b5/element_0', ml) assert_in( 'domain="wan" priority="4" client_extract="false">davs://davs.aperture.com:443/test/chamber/mock/58/b5/element_0', ml) assert_in( 'domain="wan" priority="5" client_extract="false">gsiftp://gsiftp.aperture.com:8446/test/chamber/mock/58/b5/element_0', ml) assert_not_in('priority="6"', ml) delete_replicas(rse_id=self.rse1_id, files=self.files) delete_replicas(rse_id=self.rse2_id, files=self.files) del_rse(self.rse1_id) del_rse(self.rse2_id)
def setup(self): if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): self.vo = {'vo': 'tst'} else: self.vo = {} self.rc = ReplicaClient() self.client_location_without_proxy = {'ip': '192.168.0.1', 'fqdn': 'anomalous-materials.blackmesa.com', 'site': 'BLACKMESA1'} self.rse_without_proxy = rse_name_generator() self.rse_without_proxy_id = add_rse(self.rse_without_proxy, **self.vo) add_rse_attribute(rse_id=self.rse_without_proxy_id, key='site', value='BLACKMESA1') self.client_location_with_proxy = {'ip': '10.0.1.1', 'fqdn': 'test-chamber.aperture.com', 'site': 'APERTURE1'} self.rse_with_proxy = rse_name_generator() self.rse_with_proxy_id = add_rse(self.rse_with_proxy, **self.vo) add_rse_attribute(rse_id=self.rse_with_proxy_id, key='site', value='APERTURE1') # APERTURE1 site has an internal proxy config_set('root-proxy-internal', 'APERTURE1', 'proxy.aperture.com:1094') self.files = [{'scope': InternalScope('mock', **self.vo), 'name': 'half-life_%s' % i, 'bytes': 1234, 'adler32': 'deadbeef', 'meta': {'events': 666}} for i in range(1, 4)] for rse_id in [self.rse_with_proxy_id, self.rse_without_proxy_id]: add_replicas(rse_id=rse_id, files=self.files, account=InternalAccount('root', **self.vo), ignore_availability=True) add_protocol(self.rse_without_proxy_id, {'scheme': 'root', 'hostname': 'root.blackmesa.com', 'port': 1409, 'prefix': '//training/facility/', 'impl': 'rucio.rse.protocols.xrootd.Default', 'domains': { 'lan': {'read': 1, 'write': 1, 'delete': 1}, 'wan': {'read': 1, 'write': 1, 'delete': 1}}}) add_protocol(self.rse_with_proxy_id, {'scheme': 'root', 'hostname': 'root.aperture.com', 'port': 1409, 'prefix': '//test/chamber/', 'impl': 'rucio.rse.protocols.xrootd.Default', 'domains': { 'lan': {'read': 1, 'write': 1, 'delete': 1}, 'wan': {'read': 1, 'write': 1, 'delete': 1}}})
class TestROOTProxy(object): def setup(self): if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): self.vo = {'vo': 'tst'} else: self.vo = {} self.rc = ReplicaClient() self.client_location_without_proxy = {'ip': '192.168.0.1', 'fqdn': 'anomalous-materials.blackmesa.com', 'site': 'BLACKMESA1'} self.rse_without_proxy = rse_name_generator() self.rse_without_proxy_id = add_rse(self.rse_without_proxy, **self.vo) add_rse_attribute(rse_id=self.rse_without_proxy_id, key='site', value='BLACKMESA1') self.client_location_with_proxy = {'ip': '10.0.1.1', 'fqdn': 'test-chamber.aperture.com', 'site': 'APERTURE1'} self.rse_with_proxy = rse_name_generator() self.rse_with_proxy_id = add_rse(self.rse_with_proxy, **self.vo) add_rse_attribute(rse_id=self.rse_with_proxy_id, key='site', value='APERTURE1') # APERTURE1 site has an internal proxy config_set('root-proxy-internal', 'APERTURE1', 'proxy.aperture.com:1094') self.files = [{'scope': InternalScope('mock', **self.vo), 'name': 'half-life_%s' % i, 'bytes': 1234, 'adler32': 'deadbeef', 'meta': {'events': 666}} for i in range(1, 4)] for rse_id in [self.rse_with_proxy_id, self.rse_without_proxy_id]: add_replicas(rse_id=rse_id, files=self.files, account=InternalAccount('root', **self.vo), ignore_availability=True) add_protocol(self.rse_without_proxy_id, {'scheme': 'root', 'hostname': 'root.blackmesa.com', 'port': 1409, 'prefix': '//training/facility/', 'impl': 'rucio.rse.protocols.xrootd.Default', 'domains': { 'lan': {'read': 1, 'write': 1, 'delete': 1}, 'wan': {'read': 1, 'write': 1, 'delete': 1}}}) add_protocol(self.rse_with_proxy_id, {'scheme': 'root', 'hostname': 'root.aperture.com', 'port': 1409, 'prefix': '//test/chamber/', 'impl': 'rucio.rse.protocols.xrootd.Default', 'domains': { 'lan': {'read': 1, 'write': 1, 'delete': 1}, 'wan': {'read': 1, 'write': 1, 'delete': 1}}}) def tearDown(self): for rse_id in [self.rse_with_proxy_id, self.rse_without_proxy_id]: delete_replicas(rse_id=rse_id, files=self.files) del_rse(self.rse_with_proxy_id) del_rse(self.rse_without_proxy_id) def test_client_list_replicas(self): """ ROOT (CLIENT): Test internal proxy prepend """ # no proxy involved replicas = [r for r in self.rc.list_replicas(dids=[{'scope': 'mock', 'name': f['name'], 'type': 'FILE'} for f in self.files], rse_expression=self.rse_without_proxy, client_location=self.client_location_without_proxy)] expected_pfns = ['root://root.blackmesa.com:1409//training/facility/mock/c9/df/half-life_1', 'root://root.blackmesa.com:1409//training/facility/mock/c1/8d/half-life_2', 'root://root.blackmesa.com:1409//training/facility/mock/16/30/half-life_3'] found_pfns = [replica['pfns'].keys()[0] for replica in replicas] assert_equal(sorted(found_pfns), sorted(expected_pfns)) # outgoing proxy needs to be prepended replicas = [r for r in self.rc.list_replicas(dids=[{'scope': 'mock', 'name': f['name'], 'type': 'FILE'} for f in self.files], rse_expression=self.rse_without_proxy, client_location=self.client_location_with_proxy)] expected_pfns = ['root://proxy.aperture.com:1094//root://root.blackmesa.com:1409//training/facility/mock/c9/df/half-life_1', 'root://proxy.aperture.com:1094//root://root.blackmesa.com:1409//training/facility/mock/c1/8d/half-life_2', 'root://proxy.aperture.com:1094//root://root.blackmesa.com:1409//training/facility/mock/16/30/half-life_3'] found_pfns = [replica['pfns'].keys()[0] for replica in replicas] assert_equal(sorted(found_pfns), sorted(expected_pfns)) # outgoing proxy at destination does not matter replicas = [r for r in self.rc.list_replicas(dids=[{'scope': 'mock', 'name': f['name'], 'type': 'FILE'} for f in self.files], rse_expression=self.rse_with_proxy, client_location=self.client_location_without_proxy)] expected_pfns = ['root://root.aperture.com:1409//test/chamber/mock/c9/df/half-life_1', 'root://root.aperture.com:1409//test/chamber/mock/c1/8d/half-life_2', 'root://root.aperture.com:1409//test/chamber/mock/16/30/half-life_3'] found_pfns = [replica['pfns'].keys()[0] for replica in replicas] assert_equal(sorted(found_pfns), sorted(expected_pfns)) # outgoing proxy does not matter when staying at site replicas = [r for r in self.rc.list_replicas(dids=[{'scope': 'mock', 'name': f['name'], 'type': 'FILE'} for f in self.files], rse_expression=self.rse_with_proxy, client_location=self.client_location_with_proxy)] expected_pfns = ['root://root.aperture.com:1409//test/chamber/mock/c9/df/half-life_1', 'root://root.aperture.com:1409//test/chamber/mock/c1/8d/half-life_2', 'root://root.aperture.com:1409//test/chamber/mock/16/30/half-life_3'] found_pfns = [replica['pfns'].keys()[0] for replica in replicas] assert_equal(sorted(found_pfns), sorted(expected_pfns)) def test_redirect_metalink_list_replicas(self): """ ROOT (REDIRECT REST): Test internal proxy prepend with metalink""" mw = [] # default behaviour - no location -> no proxy res = TestApp(redirect_app.wsgifunc(*mw)).get('/mock/half-life_1/metalink', expect_errors=True) assert_in('root://root.blackmesa.com:1409//training/facility/mock/c9/df/half-life_1', res.body) assert_in('root://root.aperture.com:1409//test/chamber/mock/c9/df/half-life_1', res.body) assert_not_in('proxy', res.body) res = TestApp(redirect_app.wsgifunc(*mw)).get('/mock/half-life_2/metalink', expect_errors=True) assert_in('root://root.blackmesa.com:1409//training/facility/mock/c1/8d/half-life_2', res.body) assert_in('root://root.aperture.com:1409//test/chamber/mock/c1/8d/half-life_2', res.body) assert_not_in('proxy', res.body) res = TestApp(redirect_app.wsgifunc(*mw)).get('/mock/half-life_3/metalink', expect_errors=True) assert_in('root://root.blackmesa.com:1409//training/facility/mock/16/30/half-life_3', res.body) assert_in('root://root.aperture.com:1409//test/chamber/mock/16/30/half-life_3', res.body) assert_not_in('proxy', res.body) # site without proxy res = TestApp(redirect_app.wsgifunc(*mw)).get('/mock/half-life_1/metalink?%s' % urlencode(self.client_location_without_proxy), expect_errors=True) assert_in('root://root.blackmesa.com:1409//training/facility/mock/c9/df/half-life_1', res.body) assert_in('root://root.aperture.com:1409//test/chamber/mock/c9/df/half-life_1', res.body) assert_not_in('proxy', res.body) res = TestApp(redirect_app.wsgifunc(*mw)).get('/mock/half-life_2/metalink?%s' % urlencode(self.client_location_without_proxy), expect_errors=True) assert_in('root://root.blackmesa.com:1409//training/facility/mock/c1/8d/half-life_2', res.body) assert_in('root://root.aperture.com:1409//test/chamber/mock/c1/8d/half-life_2', res.body) assert_not_in('proxy', res.body) res = TestApp(redirect_app.wsgifunc(*mw)).get('/mock/half-life_3/metalink?%s' % urlencode(self.client_location_without_proxy), expect_errors=True) assert_in('root://root.blackmesa.com:1409//training/facility/mock/16/30/half-life_3', res.body) assert_in('root://root.aperture.com:1409//test/chamber/mock/16/30/half-life_3', res.body) assert_not_in('proxy', res.body) # at location with outgoing proxy, prepend for wan replica res = TestApp(redirect_app.wsgifunc(*mw)).get('/mock/half-life_1/metalink?%s' % urlencode(self.client_location_with_proxy), expect_errors=True) assert_in('root://proxy.aperture.com:1094//root://root.blackmesa.com:1409//training/facility/mock/c9/df/half-life_1', res.body) assert_in('root://root.aperture.com:1409//test/chamber/mock/c9/df/half-life_1', res.body) res = TestApp(redirect_app.wsgifunc(*mw)).get('/mock/half-life_2/metalink?%s' % urlencode(self.client_location_with_proxy), expect_errors=True) assert_in('root://proxy.aperture.com:1094//root://root.blackmesa.com:1409//training/facility/mock/c1/8d/half-life_2', res.body) assert_in('root://root.aperture.com:1409//test/chamber/mock/c1/8d/half-life_2', res.body) res = TestApp(redirect_app.wsgifunc(*mw)).get('/mock/half-life_3/metalink?%s' % urlencode(self.client_location_with_proxy), expect_errors=True) assert_in('root://proxy.aperture.com:1094//root://root.blackmesa.com:1409//training/facility/mock/16/30/half-life_3', res.body) assert_in('root://root.aperture.com:1409//test/chamber/mock/16/30/half-life_3', res.body)
}, # {'wildcard': {'filename': 'EVNT*'}}, # {'wildcard': {'event': 'get_sm*'}}, { 'term': { 'event': 'get_sm' } }, # {'term': {'event': 'get_sm_a'}}, # {'term': {'event': 'download'}}, ] } } } rc = ReplicaClient() scroll = scan(client=es, index="traces", query=my_query) count = 0 requests = [] for res in scroll: r = res['_source'] # print(r) scope = r['scope'] filen = r['filename'] files = r['filesize'] times = r['time_start'] * 1000 gen = rc.list_replicas(dids=[{ 'scope': scope, 'name': filen }],
def setup(self): self.rc = ReplicaClient() self.client_location_without_proxy = { 'ip': '192.168.0.1', 'fqdn': 'anomalous-materials.blackmesa.com', 'site': 'BLACKMESA1' } self.rse_without_proxy = rse_name_generator() add_rse(self.rse_without_proxy) add_rse_attribute(rse=self.rse_without_proxy, key='site', value='BLACKMESA1') self.client_location_with_proxy = { 'ip': '10.0.1.1', 'fqdn': 'test-chamber.aperture.com', 'site': 'APERTURE1' } self.rse_with_proxy = rse_name_generator() add_rse(self.rse_with_proxy) add_rse_attribute(rse=self.rse_with_proxy, key='root-proxy-internal', value='root://proxy.aperture.com:1094') add_rse_attribute(rse=self.rse_with_proxy, key='site', value='APERTURE1') self.files = [{ 'scope': 'mock', 'name': 'half-life_%s' % i, 'bytes': 1234, 'adler32': 'deadbeef', 'meta': { 'events': 666 } } for i in range(1, 4)] for rse in [self.rse_with_proxy, self.rse_without_proxy]: add_replicas(rse=rse, files=self.files, account='root', ignore_availability=True) add_protocol( self.rse_without_proxy, { 'scheme': 'root', 'hostname': 'root.blackmesa.com', 'port': 1409, 'prefix': '//training/facility/', 'impl': 'rucio.rse.protocols.xrootd.Default', 'domains': { 'lan': { 'read': 1, 'write': 1, 'delete': 1 }, 'wan': { 'read': 1, 'write': 1, 'delete': 1 } } }) add_protocol( self.rse_with_proxy, { 'scheme': 'root', 'hostname': 'root.aperture.com', 'port': 1409, 'prefix': '//test/chamber/', 'impl': 'rucio.rse.protocols.xrootd.Default', 'domains': { 'lan': { 'read': 1, 'write': 1, 'delete': 1 }, 'wan': { 'read': 1, 'write': 1, 'delete': 1 } } })
except pika.exceptions.AMQPConnectionError as eek: rabbitmq = None retry_count += 1 if retry_count < retries: print( "Failed to connect to RabbitMQ. Waiting before trying again" ) time.sleep(retry_interval) else: print("Failed to connect to RabbitMQ. Giving Up") raise eek # Main Script args = parser.parse_args() site = args.site prefix = args.prefix threads = args.threads print("--------- ServiceX DID Finder ----------------") print("Threads: " + str(threads)) print("Site: " + str(site)) print("Prefix: " + str(prefix)) did_client = DIDClient() replica_client = ReplicaClient() rucio_adapter = RucioAdapter(did_client, replica_client) init_rabbit_mq(args.rabbit_uri, retries=12, retry_interval=10)