def get_volatile(self, rse):
if rse in self.__rses:
return self.__rses[rse]
rse_attributes = get_rse(rse)
self.__rses[rse] = rse_attributes["volatile"]
return self.__rses[rse]
def make_replicas_available(self):
"""
Marks available replicas for the dataset at rse if they are in PhEDEx
"""
with monitor.record_timer_block('cms_sync.time_recover_replica'):
logging.info('Recovering unavailable replicas for %s:%s at %s',
self.scope, self.block_name, self.rse)
replicas = list_replicas(dids=[{
'scope': self.scope,
'name': self.block_name
}],
rse_expression='rse=%s' % self.rse,
all_states=True)
try:
unavailable_replicas = {
repl['name']
for repl in replicas
if repl['states'][self.rse] != 'AVAILABLE'
}
except TypeError:
unavailable_replicas = set()
phedex_replicas = set(self.replicas.keys())
missing = list(phedex_replicas & unavailable_replicas)
logging.info(
'Recovery for %s:%s at %s: PhEDEx has %s, Rucio unavailable %s. Missing: %s ',
self.scope, self.block_name, self.rse, len(phedex_replicas),
len(unavailable_replicas), len(missing))
# Fix up things which are unavailable
rse_details = get_rse(self.rse)
rse_id = rse_details['id']
scope = InternalScope(self.scope)
state = 'A'
for name in missing:
logging.info('Setting available %s:%s at %s', self.scope, name,
self.rse)
core_update_state(rse_id=rse_id,
scope=scope,
name=name,
state=state)
monitor.record_counter('cms_sync.files_made_available',
delta=len(missing))
return
def __init__(self,
block_name,
pnn,
rse=None,
lifetime=None,
dry_run=False):
"""
Get the status of replica of pditem at pnn
considering only closed blocks completely replicated at site.
:rds: PhEDEx block name.
:pnn: PhEDEx node name.
:rse: Rucio RSE. If None (default) inferred by the pnn using DEFAULT_RSE_FMT.
:scope: Scope. Default: DEFAULT_SCOPE.
"""
self.phedex_svc = PhEDEx()
self.dry_run = dry_run
self.pnn = pnn
if rse is None:
self.rse = list_rses('cms_type=real&pnn=%s' % self.pnn)[0]['rse']
else:
self.rse = rse
rse_details = get_rse(self.rse)
self.rse_id = rse_details['id']
self.account = (SYNC_ACCOUNT_FMT % self.rse.lower())[:25]
self.container = self.phedex_svc.check_data_item(
pditem=block_name)['pds']
self.scope = DEFAULT_SCOPE
self.block_name = block_name
self.lifetime = lifetime
self.group, self.custodial, self.is_at_pnn = self.phedex_svc.block_at_pnn_phedex(
block=self.block_name, pnn=self.pnn)
self.block_in_phedex = self.phedex_svc.block_exists(
block=self.block_name)
self.block_known = self.phedex_svc.block_known(block=self.block_name)
if self.is_at_pnn:
self.replicas = self.phedex_svc.fileblock_files_phedex(
pnn=pnn, pfb=block_name)
else:
self.replicas = {}
self.container_exists = None
self.block_exists = None
self.rule_exists = None
touch(text=self.rse)
def GET(self, rse):
""" Details about a specific RSE.
HTTP Success:
200 OK
HTTP Error:
401 Unauthorized
404 Resource not Found
500 InternalError
:returns: A list containing all RSEs.
"""
header('Content-Type', 'application/json')
try:
rse_prop = get_rse(rse=rse)
return render_json(**rse_prop)
except RSENotFound, error:
raise generate_http_error(404, 'RSENotFound', error[0][0])
def get(self, rse):
""" Details about a specific RSE.
.. :quickref: RSE; get RSE details.
:param rse: the RSE name.
:resheader Content-Type: application/json
:status 200: OK.
:status 401: Invalid Auth Token.
:status 404: RSE not found.
:status 406: Not Acceptable.
:returns: A list containing all RSEs.
"""
try:
rse_prop = get_rse(rse=rse, vo=request.environ.get('vo'))
return Response(render_json(**rse_prop), content_type="application/json")
except RSENotFound as error:
return generate_http_error_flask(404, error)
def GET(self, rse):
""" Details about a specific RSE.
HTTP Success:
200 OK
HTTP Error:
401 Unauthorized
404 Resource not Found
500 InternalError
:returns: A list containing all RSEs.
"""
header('Content-Type', 'application/json')
try:
rse_prop = get_rse(rse=rse)
return render_json(**rse_prop)
except RSENotFound, e:
raise generate_http_error(404, 'RSENotFound', e[0][0])
def get(self, rse):
""" Details about a specific RSE.
.. :quickref: RSE; get RSE details.
:param rse: the RSE name.
:resheader Content-Type: application/json
:status 200: OK.
:status 401: Invalid Auth Token.
:status 404: RSE not found.
:status 500: Internal Error.
:returns: A list containing all RSEs.
"""
try:
rse_prop = get_rse(rse=rse)
return Response(render_json(**rse_prop),
content_type="application/json")
except RSENotFound, error:
return generate_http_error_flask(404, 'RSENotFound', error.args[0])
def GET(self, rse):
""" Details about a specific RSE.
HTTP Success:
200 OK
HTTP Error:
401 Unauthorized
404 Resource not Found
406 Not Acceptable
500 InternalError
:returns: A list containing all RSEs.
"""
header('Content-Type', 'application/json')
try:
rse_prop = get_rse(rse=rse, vo=ctx.env.get('vo'))
return render_json(**rse_prop)
except RSENotFound as error:
raise generate_http_error(404, 'RSENotFound', error.args[0])
except RucioException as error:
raise generate_http_error(500, error.__class__.__name__, error.args[0])
def test_api_rse(self):
""" RSE (API): Test external representation of RSEs """
out = api_rse.get_rse(self.rse_name, **self.vo)
assert out['rse'] == self.rse_name
assert out['id'] == self.rse_id
out = api_rse.list_rses(**self.new_vo)
out = list(out)
assert 0 != len(out)
rse_ids = [rse['id'] for rse in out]
assert self.rse3_id in rse_ids
assert self.rse4_id in rse_ids
for rse in out:
assert 'rse' in rse
if rse['id'] == self.rse3_id:
assert rse['rse'] == self.rse3_name
elif rse['id'] == self.rse4_id:
assert rse['rse'] == self.rse4_name
key = "KEY_" + generate_uuid()
api_rse.add_rse_attribute(self.rse_name,
key,
1,
issuer='root',
**self.vo)
out = api_rse.get_rses_with_attribute(key)
out = list(out)
assert 0 != len(out)
for rse in out:
assert rse['rse'] == self.rse_name
out = api_rse.get_rse_protocols(self.rse_name,
issuer='root',
**self.vo)
assert out['rse'] == self.rse_name
# add some account and RSE counters
rse_mock = 'MOCK4'
rse_mock_id = get_rse_id(rse_mock, **self.vo)
account_counter.del_counter(rse_id=rse_mock_id, account=self.account)
account_counter.add_counter(rse_id=rse_mock_id, account=self.account)
account_counter.increase(rse_id=rse_mock_id,
account=self.account,
files=1,
bytes=10)
account_counter.update_account_counter(self.account, rse_mock_id)
did = 'file_' + generate_uuid()
add_did(self.scope_name,
did,
'DATASET',
'root',
account=self.account_name,
rse=rse_mock,
**self.vo)
abacus_rse.run(once=True)
out = api_rse.get_rse_usage(rse_mock,
per_account=True,
issuer='root',
**self.vo)
assert rse_mock_id in [o['rse_id'] for o in out]
for usage in out:
if usage['rse_id'] == rse_mock_id:
assert usage['rse'] == rse_mock
accounts = [u['account'] for u in usage['account_usages']]
assert self.account_name in accounts
if self.multi_vo:
assert self.account.internal not in accounts
# clean up files
cleaner.run(once=True)
if self.multi_vo:
reaper.run(once=True,
include_rses='vo=%s&(%s)' % (self.vo['vo'], rse_mock),
greedy=True)
else:
reaper.run(once=True, include_rses=rse_mock, greedy=True)
abacus_rse.run(once=True)
out = api_rse.parse_rse_expression(
'%s|%s' % (self.rse_name, self.rse2_name), **self.vo)
assert self.rse_name in out
assert self.rse2_name in out
assert self.rse_id not in out
assert self.rse2_id not in out