class RLFSMPhEDExReserveCopyInterface(CopyInterface):
"""
CopyInterface using the Dynamo RLFSM.
"""
def __init__(self, config = None):
CopyInterface.__init__(self, config)
self.rlfsm = RLFSM(config.get('rlfsm', None))
self.mysql = MySQL(config.reserve_db_params)
def set_read_only(self, value = True): #override
self._read_only = value
self.rlfsm.set_read_only(value)
def schedule_copies(self, replica_list, operation_id, comments = ''): #override
sites = set(r.site for r in replica_list)
if len(sites) != 1:
raise OperationalError('schedule_copies should be called with a list of replicas at a single site.')
LOG.info('Scheduling copy of %d replicas to %s using RLFSM (operation %d)', len(replica_list), list(sites)[0], operation_id)
result = []
for replica in replica_list:
# Function spec is to return clones (so that if specific block fails to copy, we can return a dataset replica without the block)
clone_replica = DatasetReplica(replica.dataset, replica.site)
clone_replica.copy(replica)
result.append(clone_replica)
for block_replica in replica.block_replicas:
LOG.debug('Subscribing files for %s', str(block_replica))
if block_replica.file_ids is None:
LOG.debug('No file to subscribe for %s', str(block_replica))
return
all_files = block_replica.block.files
missing_files = all_files - block_replica.files()
for lfile in missing_files:
self.rlfsm.subscribe_file(block_replica.site, lfile)
clone_block_replica = BlockReplica(block_replica.block, block_replica.site, block_replica.group)
clone_block_replica.copy(block_replica)
clone_block_replica.last_update = int(time.time())
clone_replica.block_replicas.add(clone_block_replica)
if not self._read_only:
for clone_replica in result:
if clone_replica.growing:
self.mysql.query('INSERT INTO `phedex_transfer_reservations` (`operation_id`, `item`, `site`, `group`) VALUES (%s, %s, %s, %s)', operation_id, clone_replica.dataset.name, clone_replica.site.name, clone_replica.group.name)
else:
for block_replica in clone_replica.block_replicas:
self.mysql.query('INSERT INTO `phedex_transfer_reservations` (`operation_id`, `item`, `site`, `group`) VALUES (%s, %s, %s, %s)', operation_id, block_replica.block.full_name(), clone_replica.site.name, block_replica.group.name)
# no external dependency - everything is a success
return result
def transfer(site, files):
"""
Requests a transfer for files from other sites
:param str site: The target site for the transfer
:param list files: List of file LFNs to transfer
:returns: Two lists of files.
The first list is of files that were not on another disk.
The second list is of files that were also not on tape.
:rtype: list, list
"""
rlfsm = RLFSM()
siteobj = inventory.sites[site]
no_disk = []
unrecoverable = []
for line in files:
path = line.strip()
fileobj = inventory.find_file(path)
ondisk = False
ontape = False
for repl in fileobj.block.replicas:
if repl.site == siteobj:
continue
if not repl.has_file(fileobj):
continue
if repl.site.storage_type == Site.TYPE_DISK:
ondisk = True
elif repl.site.storage_type == Site.TYPE_MSS:
ontape = True
if not ondisk:
no_disk.append(line)
if not ontape:
unrecoverable.append(line)
if ondisk or ontape:
rlfsm.subscribe_file(siteobj, fileobj)
LOG.info('Copying %s', path)
rlfsm.db.close()
return no_disk, unrecoverable
class RLFSMCopyInterface(CopyInterface):
"""
CopyInterface using the Dynamo RLFSM.
"""
def __init__(self, config = None):
CopyInterface.__init__(self, config)
self.rlfsm = RLFSM(config.get('rlfsm', None))
def set_read_only(self, value = True): #override
self._read_only = value
self.rlfsm.set_read_only(value)
def schedule_copies(self, replica_list, operation_id, comments = ''): #override
sites = set(r.site for r in replica_list)
if len(sites) != 1:
raise OperationalError('schedule_copies should be called with a list of replicas at a single site.')
LOG.info('Scheduling copy of %d replicas to %s using RLFSM (operation %d)', len(replica_list), list(sites)[0], operation_id)
result = []
for replica in replica_list:
# Function spec is to return clones (so that if specific block fails to copy, we can return a dataset replica without the block)
clone_replica = DatasetReplica(replica.dataset, replica.site)
clone_replica.copy(replica)
result.append(clone_replica)
for block_replica in replica.block_replicas:
LOG.debug('Subscribing files for %s', str(block_replica))
if block_replica.file_ids is None:
LOG.debug('No file to subscribe for %s', str(block_replica))
return
all_files = block_replica.block.files
missing_files = all_files - block_replica.files()
for lfile in missing_files:
self.rlfsm.subscribe_file(block_replica.site, lfile)
clone_block_replica = BlockReplica(block_replica.block, block_replica.site, block_replica.group)
clone_block_replica.copy(block_replica)
clone_block_replica.last_update = int(time.time())
clone_replica.block_replicas.add(clone_block_replica)
# no external dependency - everything is a success
return result