def register_dataset(self, dry=False):
"""
Register the dataset (if there is a replica at the pnn)
:dry: Dry run. Default false.
"""
try:
self.rcli.get_did(scope=self.scope, name=self.dataset)
return 'exists'
except DataIdentifierNotFound:
pass
if self.is_at_pnn and dry:
logging.dry('Create dataset %s in scope %s.', self.dataset,
self.scope)
return 'created'
elif self.is_at_pnn:
logging.verbose('Create dataset %s in scope %s.', self.dataset,
self.scope)
self.rcli.add_dataset(scope=self.scope,
name=self.dataset,
lifetime=self.lifetime)
self.rcli.attach_dids(scope=self.scope,
name=self.container,
dids=[{
'scope': self.scope,
'name': self.dataset
}])
return 'created'
return 'skipped'
def register_container(self, dry=False):
"""
Register container of the dataset
(only if there is a dataset replica on the pnn)
:dry: Dry run. Default false.
"""
try:
self.rcli.get_did(scope=self.scope, name=self.container)
return 'exists'
except DataIdentifierNotFound:
pass
if self.is_at_pnn and dry:
logging.dry('Create container %s in scope %s.', self.container,
self.scope)
return 'created'
elif self.is_at_pnn:
logging.verbose('Create container %s in scope %s.', self.container,
self.scope)
try:
self.rcli.add_container(scope=self.scope,
name=self.container,
lifetime=self.lifetime)
except DataIdentifierAlreadyExists:
logging.warning('Container was created in the meanwhile')
return 'exists'
return 'created'
return 'skipped'
def update_rule(self, dry=False):
"""
Adds or removes the rule for the dataset.
:dry: Drydrun. default false
returns the action performed: None, added, removed
"""
rules = self.rcli.list_did_rules(scope=self.scope, name=self.dataset)
rrule = None
account = self.rcli.__dict__['account']
action = None
rse_exp = 'rse=' + self.rse
rrule = next((
rule for rule in rules
if rule['account'] == account and\
rule['rse_expression'] == rse_exp
), None)
if rrule is None and self.is_at_pnn:
if dry:
logging.dry("Adding rule for dataset %s at rse %s.",
self.dataset, self.rse)
else:
self.rcli.add_replication_rule(
dids=[{
'scope': self.scope,
'name': self.dataset
}],
copies=1,
rse_expression=rse_exp,
)
action = 'added'
elif rrule is not None and not self.is_at_pnn:
# removing rule
if dry:
logging.dry("Removing rule for dataset %s at rse %s.",
self.dataset, self.rse)
else:
self.rcli.delete_replication_rule(rrule['id'],
purge_replicas=False)
action = 'removed'
return action
def update_replicas(self, dry=False):
"""
Add or removes replicas for the dataset at rse.
:dry: Drydrun. default false
"""
logging.notice('Updating replicas for %s:%s at %s' %
(self.scope, self.dataset, self.rse))
replicas = self.rcli.list_replicas([{
'scope': self.scope,
'name': self.dataset
}],
rse_expression='rse=%s' % self.rse)
rrepl = [repl['name'] for repl in replicas]
prepl = [repl for repl in self.replicas.keys()]
missing = list(set(prepl) - set(rrepl))
to_remove = list(set(rrepl) - set(prepl))
if missing and dry:
logging.dry('Adding replicas %s to rse %s.', str(missing),
self.rse)
elif missing:
logging.verbose('Adding replicas %s to rse %s.', str(missing),
self.rse)
self.rcli.add_replicas(rse=self.rse,
files=[{
'scope':
self.scope,
'name':
self.replicas[lfn]['name'],
'adler32':
self.replicas[lfn]['checksum'],
'bytes':
self.replicas[lfn]['size'],
} for lfn in missing])
# missing files that are not in the list of dataset files
# are to be attached.
lfns = [
item['name']
for item in self.rcli.list_files(scope=self.scope,
name=self.dataset)
]
missing_lfns = list(set(missing) - set(lfns))
if missing_lfns:
logging.verbose('Attaching lfns %s to dataset %s.',
str(missing_lfns), self.dataset)
try:
self.rcli.attach_dids(
scope=self.scope,
name=self.dataset,
dids=[{
'scope': self.scope,
'name': lfn
} for lfn in list(set(missing) - set(lfns))])
except FileAlreadyExists:
logging.warning('Trying to attach already existing files.')
if to_remove and dry:
logging.dry('Removing replicas %s from rse %s.', str(to_remove),
self.rse)
elif to_remove:
logging.verbose('Removing replicas %s from rse %s.',
str(to_remove), self.rse)
for to_remove_chunk in chunks(to_remove, REMOVE_CHUNK_SIZE):
attempt = 0
while True:
attempt += 1
try:
self.rcli.delete_replicas(rse=self.rse,
files=[{
'scope': self.scope,
'name': lfn,
} for lfn in to_remove_chunk
])
break
except DatabaseException:
logging.warning(
'DatabaseException raised, retrying...')
if attempt > 3:
raise
time.sleep(randint(1, 5))
return {'added': missing, 'removed': to_remove}