def add_sync_rule(rse, scope, block, account, session=None):
rse_id = get_rse_id(rse=rse, session=session)
scope = InternalScope(scope)
account = InternalAccount(account)
files = []
for file in list_files(scope, block, long=False, session=session):
try:
update_replicas_states(
replicas=[
{
"scope": scope,
"name": file["name"],
"rse_id": rse_id,
"state": "A",
}
],
add_tombstone=False,
session=session,
)
except ReplicaNotFound:
add_replicas(
rse_id=rse_id,
files=[
{
"scope": scope,
"name": file["name"],
"bytes": file["bytes"],
"adler32": file["adler32"],
}
],
account=account,
ignore_availability=True,
session=session,
)
rules = add_rule(
dids=[{"scope": scope, "name": block}],
copies=1,
rse_expression=rse,
weight=None,
lifetime=None,
grouping="DATASET",
account=account,
locked=False,
subscription_id=None,
source_replica_expression=None,
activity="Data Consolidation",
notify=None,
purge_replicas=False,
ignore_availability=True,
comment=None,
ask_approval=False,
asynchronous=False,
priority=3,
split_container=False,
meta=json.dumps({"phedex_group": "DataOps", "phedex_custodial": True}),
session=session,
)
return rules[0]
def list_files(scope, name, long):
"""
List data identifier file contents.
:param scope: The scope name.
:param name: The data identifier name.
:param long: A boolean to choose if GUID is returned or not.
"""
return did.list_files(scope=scope, name=name, long=long)
def list_files(scope, name, long):
"""
List data identifier file contents.
:param scope: The scope name.
:param name: The data identifier name.
:param long: A boolean to choose if GUID is returned or not.
"""
return did.list_files(scope=scope, name=name, long=long)
def list_files(scope, name, long):
"""
List data identifier file contents.
:param scope: The scope name.
:param name: The data identifier name.
:param long: A boolean to choose if GUID is returned or not.
"""
scope = InternalScope(scope)
dids = did.list_files(scope=scope, name=name, long=long)
for d in dids:
yield api_update_return_dict(d)
def test_delete_replicas_from_datasets(self):
""" REPLICA (CORE): Delete replicas from dataset """
tmp_scope = 'mock'
tmp_dsn1 = 'dsn_%s' % generate_uuid()
tmp_dsn2 = 'dsn_%s' % generate_uuid()
nbfiles = 5
files1 = [{
'scope': tmp_scope,
'name': 'file_%s' % generate_uuid(),
'bytes': 1,
'adler32': '0cc737eb',
'meta': {
'events': 10
}
} for _ in range(nbfiles)]
add_did(scope=tmp_scope,
name=tmp_dsn1,
type=DIDType.DATASET,
account='root')
add_did(scope=tmp_scope,
name=tmp_dsn2,
type=DIDType.DATASET,
account='root')
attach_dids(scope=tmp_scope,
name=tmp_dsn1,
rse='MOCK',
dids=files1,
account='root')
attach_dids(scope=tmp_scope,
name=tmp_dsn2,
dids=files1,
account='root')
set_status(scope=tmp_scope, name=tmp_dsn1, open=False)
delete_replicas(rse='MOCK', files=files1)
with assert_raises(DataIdentifierNotFound):
get_did(scope=tmp_scope, name=tmp_dsn1)
get_did(scope=tmp_scope, name=tmp_dsn2)
assert_equal([f for f in list_files(scope=tmp_scope, name=tmp_dsn2)],
[])
def list_files(scope, name, long, vo='def', session=None):
"""
List data identifier file contents.
:param scope: The scope name.
:param name: The data identifier name.
:param long: A boolean to choose if GUID is returned or not.
:param vo: The VO to act on.
:param session: The database session in use.
"""
scope = InternalScope(scope, vo=vo)
dids = did.list_files(scope=scope, name=name, long=long, session=session)
for d in dids:
yield api_update_return_dict(d, session=session)
def delete_sync_rule(rule_id, session=None):
rule = get_rule(rule_id, session=session)
if rule["did_type"] != DIDType.DATASET:
raise RuntimeError("Rule applies to did with wrong type")
block = rule["name"]
try:
rse_id = get_rse_id(rse=rule["rse_expression"], session=session)
except RSENotFound:
raise RuntimeError("Rule does not apply to a specific RSE")
scope = rule["scope"]
account = rule["account"]
files = []
for file in list_files(scope, block, long=False, session=session):
files.append(
{"scope": scope, "name": file["name"], "rse_id": rse_id, "state": "U"}
)
update_replicas_states(
replicas=files, add_tombstone=False, session=session
)
delete_rule(rule_id=rule_id, purge_replicas=True, soft=False, session=session)
account='root')
attach_dids(scope=tmp_scope,
name=tmp_dsn2,
dids=files1,
account='root')
set_status(scope=tmp_scope, name=tmp_dsn1, open=False)
delete_replicas(rse='MOCK', files=files1)
with assert_raises(DataIdentifierNotFound):
get_did(scope=tmp_scope, name=tmp_dsn1)
get_did(scope=tmp_scope, name=tmp_dsn2)
assert_equal([f for f in list_files(scope=tmp_scope, name=tmp_dsn2)],
[])
def test_update_lock_counter(self):
""" RSE (CORE): Test the update of a replica lock counter """
rse = 'MOCK'
tmp_scope = 'mock'
tmp_file = 'file_%s' % generate_uuid()
add_replica(rse=rse,
scope=tmp_scope,
name=tmp_file,
bytes=1L,
adler32='0cc737eb',
account='jdoe')
values = (1, 1, 1, -1, -1, -1, 1, 1, -1)
add_did(scope=tmp_scope, name=tmp_dsn1, type=DIDType.DATASET, account='root')
add_did(scope=tmp_scope, name=tmp_dsn2, type=DIDType.DATASET, account='root')
attach_dids(scope=tmp_scope, name=tmp_dsn1, rse='MOCK', dids=files1, account='root')
attach_dids(scope=tmp_scope, name=tmp_dsn2, dids=files1, account='root')
set_status(scope=tmp_scope, name=tmp_dsn1, open=False)
delete_replicas(rse='MOCK', files=files1)
with assert_raises(DataIdentifierNotFound):
get_did(scope=tmp_scope, name=tmp_dsn1)
get_did(scope=tmp_scope, name=tmp_dsn2)
assert_equal([f for f in list_files(scope=tmp_scope, name=tmp_dsn2)], [])
def test_update_lock_counter(self):
""" RSE (CORE): Test the update of a replica lock counter """
rse = 'MOCK'
tmp_scope = 'mock'
tmp_file = 'file_%s' % generate_uuid()
add_replica(rse=rse, scope=tmp_scope, name=tmp_file, bytes=1L, adler32='0cc737eb', account='jdoe')
values = (1, 1, 1, -1, -1, -1, 1, 1, -1)
tombstones = (True, True, True, True, True, False, True, True, True)
lock_counters = (1, 2, 3, 2, 1, 0, 1, 2, 1)
for value, tombstone, lock_counter in zip(values, tombstones, lock_counters):
status = update_replica_lock_counter(rse=rse, scope=tmp_scope, name=tmp_file, value=value)
assert_equal(status, True)
replica = get_replica(rse=rse, scope=tmp_scope, name=tmp_file)