def register_dataset(self, dry=False):
"""
Register the dataset (if there is a replica at the pnn)
:dry: Dry run. Default false.
"""
try:
self.rcli.get_did(scope=self.scope, name=self.dataset)
return 'exists'
except DataIdentifierNotFound:
pass
if self.is_at_pnn and dry:
logging.dry('Create dataset %s in scope %s.', self.dataset,
self.scope)
return 'created'
elif self.is_at_pnn:
logging.verbose('Create dataset %s in scope %s.', self.dataset,
self.scope)
self.rcli.add_dataset(scope=self.scope,
name=self.dataset,
lifetime=self.lifetime)
self.rcli.attach_dids(scope=self.scope,
name=self.container,
dids=[{
'scope': self.scope,
'name': self.dataset
}])
return 'created'
return 'skipped'
def register_container(self, dry=False):
"""
Register container of the dataset
(only if there is a dataset replica on the pnn)
:dry: Dry run. Default false.
"""
try:
self.rcli.get_did(scope=self.scope, name=self.container)
return 'exists'
except DataIdentifierNotFound:
pass
if self.is_at_pnn and dry:
logging.dry('Create container %s in scope %s.', self.container,
self.scope)
return 'created'
elif self.is_at_pnn:
logging.verbose('Create container %s in scope %s.', self.container,
self.scope)
try:
self.rcli.add_container(scope=self.scope,
name=self.container,
lifetime=self.lifetime)
except DataIdentifierAlreadyExists:
logging.warning('Container was created in the meanwhile')
return 'exists'
return 'created'
return 'skipped'
def _get_pnn_workers(workers, summary):
"""
Gets the finished workers for the pnn worker
"""
for dataset, work in workers.items():
ret = work.get()
logging.verbose("Recovered worker %s, ret %s", dataset, ret)
_get_pnn_worker_ret(ret, summary)
return summary
def _ping(rcli=None):
"""
pings the Rucio server
"""
if rcli is None:
rcli = Client()
try:
rcli.ping()
except RucioException as exc:
logging.verbose('Ping Exception %s, %s',
type(exc).__name__,
traceback.format_exc().replace('\n', '~~'))
return False
return True
def _get_dset_list(pcli, datasets):
logging.verbose("Getting datasets list for: %s", datasets)
ret = []
wildcard = re.compile(r'\S*[*]\S*')
for dset in datasets:
ret.extend([
item for item in pcli.list_data_items(
pditem=dset, metadata=False, locality=False)
if not wildcard.match(item)
])
ret = list(set(ret))
logging.verbose("Got %d datasets", len(ret))
return ret
def update_replicas(self, dry=False):
"""
Add or removes replicas for the dataset at rse.
:dry: Drydrun. default false
"""
logging.notice('Updating replicas for %s:%s at %s' %
(self.scope, self.dataset, self.rse))
replicas = self.rcli.list_replicas([{
'scope': self.scope,
'name': self.dataset
}],
rse_expression='rse=%s' % self.rse)
rrepl = [repl['name'] for repl in replicas]
prepl = [repl for repl in self.replicas.keys()]
missing = list(set(prepl) - set(rrepl))
to_remove = list(set(rrepl) - set(prepl))
if missing and dry:
logging.dry('Adding replicas %s to rse %s.', str(missing),
self.rse)
elif missing:
logging.verbose('Adding replicas %s to rse %s.', str(missing),
self.rse)
self.rcli.add_replicas(rse=self.rse,
files=[{
'scope':
self.scope,
'name':
self.replicas[lfn]['name'],
'adler32':
self.replicas[lfn]['checksum'],
'bytes':
self.replicas[lfn]['size'],
} for lfn in missing])
# missing files that are not in the list of dataset files
# are to be attached.
lfns = [
item['name']
for item in self.rcli.list_files(scope=self.scope,
name=self.dataset)
]
missing_lfns = list(set(missing) - set(lfns))
if missing_lfns:
logging.verbose('Attaching lfns %s to dataset %s.',
str(missing_lfns), self.dataset)
try:
self.rcli.attach_dids(
scope=self.scope,
name=self.dataset,
dids=[{
'scope': self.scope,
'name': lfn
} for lfn in list(set(missing) - set(lfns))])
except FileAlreadyExists:
logging.warning('Trying to attach already existing files.')
if to_remove and dry:
logging.dry('Removing replicas %s from rse %s.', str(to_remove),
self.rse)
elif to_remove:
logging.verbose('Removing replicas %s from rse %s.',
str(to_remove), self.rse)
for to_remove_chunk in chunks(to_remove, REMOVE_CHUNK_SIZE):
attempt = 0
while True:
attempt += 1
try:
self.rcli.delete_replicas(rse=self.rse,
files=[{
'scope': self.scope,
'name': lfn,
} for lfn in to_remove_chunk
])
break
except DatabaseException:
logging.warning(
'DatabaseException raised, retrying...')
if attempt > 3:
raise
time.sleep(randint(1, 5))
return {'added': missing, 'removed': to_remove}
