def get(self, scope):
"""
List all data identifiers in a scope which match a given metadata.
.. :quickref: Search; Search DIDs in a scope with given metadata.
**Example request**:
.. sourcecode:: http
GET /dids/scope1/dids/search?type=collection1&long=True HTTP/1.1
Host: rucio.cern.ch
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Vary: Accept
Content-Type: application/x-json-stream
{"scope": "scope1", "did_type": "CONTAINER", "name": "container1",
"bytes": 1234, "length": 1}
{"scope": "scope1", "did_type": "DATASET", "name": "dataset1",
"bytes": 234, "length": 3}
:query type: specify a DID type to search for
:query long: set to True for long output, otherwise only name
:resheader Content-Type: application/x-json-stream
:status 200: DIDs found
:status 401: Invalid Auth Token
:status 404: Invalid key in filters
:status 409: Wrong DID type
:returns: Line separated name of DIDs or dictionaries of DIDs for long option
"""
filters = {}
long = False
type = 'collection'
for k, v in request.args.items():
if k == 'type':
type = v
elif k == 'long':
long = bool(v)
else:
filters[k] = v[0]
try:
data = ""
for did in list_dids(scope=scope,
filters=filters,
type=type,
long=long):
data += dumps(did) + '\n'
return Response(data, content_type='application/x-json-stream')
except UnsupportedOperation, error:
return generate_http_error_flask(409, 'UnsupportedOperation',
error.args[0][0])
def generate(vo):
for did in list_dids(scope=scope,
filters=filters,
type=type_param,
limit=limit,
long=long,
recursive=recursive,
vo=vo):
yield dumps(did) + '\n'
def GET(self, scope):
"""
List all data identifiers in a scope which match a given metadata.
HTTP Success:
200 OK
HTTP Error:
401 Unauthorized
404 KeyNotFound
406 Not Acceptable
409 UnsupportedOperation
:param scope: The scope name.
"""
header('Content-Type', 'application/x-json-stream')
filters = {}
limit = None
long = False
recursive = False
if ctx.query:
params = parse_qs(ctx.query[1:])
for k, v in params.items():
if k == 'type':
type = v[0]
elif k == 'limit':
limit = v[0]
elif k == 'long':
long = v[0] in ['True', '1']
elif k == 'recursive':
recursive = v[0] == 'True'
else:
filters[k] = v[0]
try:
for did in list_dids(scope=scope,
filters=filters,
type=type,
limit=limit,
long=long,
recursive=recursive,
vo=ctx.env.get('vo')):
yield dumps(did) + '\n'
except UnsupportedOperation as error:
raise generate_http_error(409, 'UnsupportedOperation',
error.args[0])
except KeyNotFound as error:
raise generate_http_error(404, 'KeyNotFound', error.args[0])
except Exception as error:
print(format_exc())
raise InternalError(error)
def test_automatix(self):
""" MULTI VO (DAEMON): Test that automatix runs on a single VO """
scope_client = ScopeClient()
scope_uuid = str(generate_uuid()).lower()[:16]
shr_scope = 'shr_%s' % scope_uuid
scope_client.add_scope('root', shr_scope)
add_scope(shr_scope, 'root', 'root', **self.new_vo)
rse_client = RSEClient()
rse_str = ''.join(choice(ascii_uppercase) for x in range(10))
shr_rse = 'SHR_%s' % rse_str
mock_protocol = {'scheme': 'MOCK',
'hostname': 'localhost',
'port': 123,
'prefix': '/test/automatix',
'impl': 'rucio.rse.protocols.mock.Default',
'domains': {
'lan': {'read': 1,
'write': 1,
'delete': 1},
'wan': {'read': 1,
'write': 1,
'delete': 1}}}
rse_client.add_rse(shr_rse)
rse_client.add_rse_attribute(rse=shr_rse, key='verify_checksum', value=False)
rse_client.add_protocol(shr_rse, mock_protocol)
add_rse(shr_rse, 'root', **self.new_vo)
add_rse_attribute(rse=shr_rse, key='verify_checksum', value=False, issuer='root', **self.new_vo)
add_protocol(rse=shr_rse, data=mock_protocol, issuer='root', **self.new_vo)
automatix(sites=[shr_rse], inputfile='/opt/rucio/etc/automatix.json', sleep_time=30, account='root', once=True, scope=shr_scope)
did_list_tst = list(DIDClient().list_dids(shr_scope, {}))
did_list_new = list(list_dids(shr_scope, {}, **self.new_vo))
assert_not_equal(len(did_list_tst), 0)
assert_equal(len(did_list_new), 0)
did_dicts = [{'scope': shr_scope, 'name': n} for n in did_list_tst]
replicas_tst = list(ReplicaClient().list_replicas(did_dicts, rse_expression=shr_rse))
replicas_new = list(list_replicas(did_dicts, rse_expression=shr_rse, **self.new_vo))
assert_not_equal(len(replicas_tst), 0)
assert_equal(len(replicas_new), 0)
def setup(self):
"""RucioCache (Func): Find necessary rse and dids """
self.id = int(random.random() * 10000)
self.rse_exist_volatile = 'RUCIO_CACHE_VOLATILE' + str(self.id)
try:
rse.add_rse(self.rse_exist_volatile, 'root', deterministic=True, volatile=True)
except exception.Duplicate:
logging.warning("rse RUCIO_CACHE_VOLATILE already there")
self.rse_exist_novolatile = 'RUCIO_CACHE_NOVOLATILE' + str(self.id)
try:
rse.add_rse(self.rse_exist_novolatile, 'root', deterministic=True, volatile=False)
except exception.Duplicate:
logging.warning("rse RUCIO_CACHE_NOVOLATILE already there")
self.rse_noExist = 'RUCIO_CACHE_NOEXIST' + str(self.id)
dids = did.list_dids(scope='mock', filters={}, type='file')
i = 0
self.files_exist = []
self.files_exist_wrong_meta = []
self.file_replica_on_novolatile = []
for _did in dids:
if i < 2:
i += 1
meta = did.get_metadata(scope='mock', name=_did[0])
self.files_exist.append({'scope': meta['scope'], 'name': meta['name'], 'bytes': meta['bytes'], "adler32": meta["adler32"]})
self.files_exist_wrong_meta.append({'scope': meta['scope'], 'name': meta['name'], 'bytes': 12345678, "adler32": '12345678'})
elif i < 3:
meta = did.get_metadata(scope='mock', name=_did[0])
file = {'scope': meta['scope'], 'name': meta['name'], 'bytes': meta['bytes'], "adler32": meta["adler32"]}
self.file_replica_on_novolatile.append(file)
replica.add_replicas(self.rse_exist_novolatile, [file], account='root')
logging.debug("File Exists: %s " % self.files_exist)
logging.debug("File Exists with wrong metadata: %s " % self.files_exist_wrong_meta)
logging.debug("File Exists on volatie rses: " % self.file_replica_on_novolatile)
self.files_noExist = [{'scope': 'mock', 'name': 'file_notexist', "bytes": 1, "adler32": "0cc737eb"}]
logging.debug("File not Exists: %s " % self.files_noExist)
self.account = 'root'
self.lifetime = 2
def test_dids_at_different_vos(self):
""" MULTI VO (CLIENT): Test that dids from 2nd vo don't interfere """
scope_uuid = str(generate_uuid()).lower()[:16]
scope = 'shr_%s' % scope_uuid
add_scope(scope, 'root', 'root', **self.vo)
add_scope(scope, 'root', 'root', **self.new_vo)
did_client = DIDClient()
did_uuid = str(generate_uuid()).lower()[:16]
tst = 'tstset_%s' % did_uuid
new = 'newset_%s' % did_uuid
shr = 'shrset_%s' % did_uuid
did_client.add_did(scope, tst, 'DATASET')
did_client.add_did(scope, shr, 'DATASET')
add_did(scope, new, 'DATASET', 'root', **self.new_vo)
add_did(scope, shr, 'DATASET', 'root', **self.new_vo)
did_list_tst = list(did_client.list_dids(scope, {}))
did_list_new = list(list_dids(scope, {}, **self.new_vo))
assert_true(tst in did_list_tst)
assert_false(new in did_list_tst)
assert_true(shr in did_list_tst)
assert_false(tst in did_list_new)
assert_true(new in did_list_new)
assert_true(shr in did_list_new)
def GET(self, scope):
"""
List all data identifiers in a scope which match a given metadata.
HTTP Success:
200 OK
HTTP Error:
401 Unauthorized
404 KeyNotFound
409 UnsupportedOperation
:param scope: The scope name.
"""
header('Content-Type', 'application/x-json-stream')
filters = {}
long = False
if ctx.query:
params = parse_qs(ctx.query[1:])
for k, v in params.items():
if k == 'type':
type = v[0]
elif k == 'long':
long = bool(v[0])
else:
filters[k] = v[0]
try:
for did in list_dids(scope=scope,
filters=filters,
type=type,
long=long):
yield dumps(did) + '\n'
except UnsupportedOperation, error:
raise generate_http_error(409, 'UnsupportedOperation',
error.args[0][0])
def setup(self):
"""RucioCache (Func): Find necessary rse and dids """
self.id = int(random.random() * 10000)
self.rse_exist_volatile = 'RUCIO_CACHE_VOLATILE' + str(self.id)
try:
rse.add_rse(self.rse_exist_volatile,
'root',
deterministic=True,
volatile=True)
except exception.Duplicate:
logging.warning("rse RUCIO_CACHE_VOLATILE already there")
self.rse_exist_novolatile = 'RUCIO_CACHE_NOVOLATILE' + str(self.id)
try:
rse.add_rse(self.rse_exist_novolatile,
'root',
deterministic=True,
volatile=False)
except exception.Duplicate:
logging.warning("rse RUCIO_CACHE_NOVOLATILE already there")
self.rse_noExist = 'RUCIO_CACHE_NOEXIST' + str(self.id)
dids = did.list_dids(scope='mock', filters={}, type='file')
i = 0
self.files_exist = []
self.files_exist_wrong_meta = []
self.file_replica_on_novolatile = []
for _did in dids:
if i < 2:
i += 1
meta = did.get_metadata(scope='mock', name=_did[0])
self.files_exist.append({
'scope': meta['scope'],
'name': meta['name'],
'bytes': meta['bytes'],
"adler32": meta["adler32"]
})
self.files_exist_wrong_meta.append({
'scope': meta['scope'],
'name': meta['name'],
'bytes': 12345678,
"adler32": '12345678'
})
elif i < 3:
meta = did.get_metadata(scope='mock', name=_did[0])
file = {
'scope': meta['scope'],
'name': meta['name'],
'bytes': meta['bytes'],
"adler32": meta["adler32"]
}
self.file_replica_on_novolatile.append(file)
replica.add_replicas(self.rse_exist_novolatile, [file],
account='root')
logging.debug("File Exists: %s " % self.files_exist)
logging.debug("File Exists with wrong metadata: %s " %
self.files_exist_wrong_meta)
logging.debug("File Exists on volatie rses: " %
self.file_replica_on_novolatile)
self.files_noExist = [{
'scope': 'mock',
'name': 'file_notexist',
"bytes": 1,
"adler32": "0cc737eb"
}]
logging.debug("File not Exists: %s " % self.files_noExist)
self.account = 'root'
self.lifetime = 2
def get(self, scope):
"""
List all data identifiers in a scope which match a given metadata.
.. :quickref: Search; Search DIDs in a scope with given metadata.
**Example request**:
.. sourcecode:: http
GET /dids/scope1/dids/search?type=collection&long=True&length.lt=10 HTTP/1.1
Host: rucio.cern.ch
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Vary: Accept
Content-Type: application/x-json-stream
{"scope": "scope1", "did_type": "CONTAINER", "name": "container1",
"bytes": 1234, "length": 1}
{"scope": "scope1", "did_type": "DATASET", "name": "dataset1",
"bytes": 234, "length": 3}
:query type: specify a DID type to search for
:query long: set to True for long output, otherwise only name
:query recursive: set to True to recursively list DIDs content
:query created_before: Date string in RFC-1123 format where the creation date was earlier
:query created_after: Date string in RFC-1123 format where the creation date was later
:query length: Exact number of attached DIDs
:query length.gt: Number of attached DIDs greater than
:query length.lt: Number of attached DIDs less than
:query length.gte: Number of attached DIDs greater than or equal to
:query length.lte: Number of attached DIDs less than or equal to
:query name: Name or pattern of a DID name
:resheader Content-Type: application/x-json-stream
:status 200: DIDs found
:status 401: Invalid Auth Token
:status 404: Invalid key in filters
:status 409: Wrong DID type
:returns: Line separated name of DIDs or dictionaries of DIDs for long option
"""
filters = {}
long = False
recursive = False
type = 'collection'
for k, v in request.args.items():
if k == 'type':
type = v
elif k == 'long':
long = v == '1'
elif k == 'recursive':
recursive = v == 'True'
else:
filters[k] = v
try:
data = ""
for did in list_dids(scope=scope,
filters=filters,
type=type,
long=long,
recursive=recursive):
data += dumps(did) + '\n'
return Response(data, content_type='application/x-json-stream')
except UnsupportedOperation as error:
return generate_http_error_flask(409, 'UnsupportedOperation',
error.args[0])
except KeyNotFound as error:
return generate_http_error_flask(404, 'KeyNotFound', error.args[0])
except Exception as error:
print(format_exc())
return error, 500
