def _commit_copies(self, cycle_number, inventory, copy_list, comment):
"""
@param cycle_number Cycle number.
@param inventory Dynamo inventory.
@param copy_list List of dataset replicas to be created (proposal - final state depends on copy scheduling success)
@param comment Comment to be passed to the copy interface.
"""
signal_blocker = SignalBlocker(logger = LOG)
by_site = collections.defaultdict(list)
for replica in copy_list:
by_site[replica.site].append(replica)
for site in sorted(by_site.iterkeys(), key = lambda s: s.name):
replicas = by_site[site]
LOG.info('Scheduling copy of %d replicas to %s.', len(replicas), site.name)
with signal_blocker:
history_record = self.history.make_cycle_entry(cycle_number, site)
scheduled_replicas = self.copy_op[site.name].schedule_copies(replicas, history_record.operation_id, comments = comment)
for replica in scheduled_replicas:
history_record.replicas.append(CopiedReplica(replica.dataset.name, replica.size(physical = False), HistoryRecord.ST_ENROUTE))
inventory.update(replica)
for block_replica in replica.block_replicas:
inventory.update(block_replica)
self.history.update_entry(history_record)
total_size = sum(r.size for r in history_record.replicas)
LOG.info('Scheduled copy %.1f TB to %s.', total_size * 1.e-12, site.name)
def _update_inventory(self, update_commands):
# My updates
self.manager.set_status(ServerHost.STAT_UPDATING)
with SignalBlocker():
self._exec_updates(update_commands)
self.manager.set_status(ServerHost.STAT_ONLINE)
# Others
self.manager.send_updates(update_commands)
def _commit_copies(self, cycle_number, inventory, copy_list, comment):
"""
@param cycle_number Cycle number.
@param inventory Dynamo inventory.
@param copy_list Flat list of dataset or block replicas.
@param comment Comment to be passed to the copy interface.
"""
signal_blocker = SignalBlocker(logger = LOG)
group = inventory.groups[self.policy.group_name]
by_site = collections.defaultdict(list)
for replica in copy_list:
by_site[replica.site].append(replica)
for site in sorted(by_site.iterkeys(), key = lambda s: s.name):
replicas = by_site[site]
LOG.info('Scheduling copy of %d replicas to %s.', len(replicas), site.name)
with signal_blocker:
copy_mapping = self.copy_op.schedule_copies(replicas, comments = comment)
# It would be better if mapping from replicas to items is somehow kept
# Then we can get rid of creating yet another replica object below, which
# means we can let each plugin to decide which group they want to make replicas in
for copy_id, (approved, site, items) in copy_mapping.iteritems():
dataset_list = set()
size = 0
for item in items:
size += item.size
if type(item) is Dataset:
dataset_list.add(item)
if approved:
replica = new_replica_from_dataset(item, site, group)
inventory.update(replica)
else:
dataset_list.add(item.dataset)
if approved:
if site.find_dataset_replica(item.dataset) is None:
replica = new_replica_from_dataset(item.dataset, site, group)
inventory.update(replica)
replica = new_replica_from_block(item, site, group)
inventory.update(replica)
self.history.make_copy_entry(cycle_number, site, copy_id, approved, dataset_list, size)
def _commit_deletions(self, cycle_number, inventory, deleted, comment):
"""
@param cycle_number Cycle number.
@param inventory Global (original) inventory
@param deleted {dataset_replica: {condition_id: set(block_replicas)}}
@param comment Comment to be passed to the deletion interface.
"""
signal_blocker = SignalBlocker(logger=LOG)
# get the original replicas from the inventory and organize them into sites
deletions_by_site = collections.defaultdict(
list) # {site: [(dataset_replica, block_replicas)]}
for replica, matches in deleted.iteritems():
site = inventory.sites[replica.site.name]
original_replica = inventory.datasets[
replica.dataset.name].find_replica(site)
original_block_replicas = dict(
(br.block.name, br) for br in original_replica.block_replicas)
all_block_replicas = set()
for block_replicas in matches.itervalues():
for block_replica in block_replicas:
all_block_replicas.add(
original_block_replicas[block_replica.block.name])
if not replica.growing and all_block_replicas == original_replica.block_replicas:
# if we are deleting all block replicas and the replica is marked as not growing, delete the DatasetReplica
deletions_by_site[site].append((original_replica, None))
else:
# otherwise delete only the BlockReplicas
deletions_by_site[site].append(
(original_replica, list(all_block_replicas)))
# now schedule deletions for each site
for site in sorted(deletions_by_site.iterkeys(), key=lambda s: s.name):
site_deletion_list = deletions_by_site[site]
LOG.info('Deleting %d replicas from %s.', len(site_deletion_list),
site.name)
null_group = inventory.groups[None]
# Block interruptions until deletion is executed and recorded
with signal_blocker:
history_record = self.history.make_cycle_entry(
cycle_number, site)
scheduled_replicas = self.deletion_op.schedule_deletions(
site_deletion_list,
history_record.operation_id,
comments=comment)
for replica, block_replicas in scheduled_replicas:
deleted_size = 0
if block_replicas is None:
replica.growing = False
replica.group = null_group
# replica is a clone -> use inventory.update instead of inventory.register_update
inventory.update(replica)
original_replica = replica.site.find_dataset_replica(
replica.dataset)
for block_replica in original_replica.block_replicas:
block_replica.group = null_group
inventory.register_update(block_replica)
deleted_size += original_replica.size()
else:
for block_replica in block_replicas:
block_replica.group = null_group
inventory.update(block_replica)
deleted_size += block_replica.size
history_record.replicas.append(
DeletedReplica(replica.dataset.name, deleted_size))
self.history.update_entry(history_record)
total_size = sum(r.size for r in history_record.replicas)
LOG.info('Done deleting %.1f TB from %s.', total_size * 1.e-12,
site.name)
def run(self):
"""
Infinite-loop main body of the daemon.
Step 1: Poll the registry for one uploaded script.
Step 2: If a script is found, check the authorization of the script.
Step 3: Spawn a child process for the script.
Step 4: Collect updates from the write-enabled child process.
Step 5: Collect completed child processes.
Step 6: Sleep for N seconds.
"""
LOG.info('Started dynamo daemon.')
child_processes = []
# There can only be one child process with write access at a time. We pass it a Queue to communicate back.
# writing_process is a tuple (proc, queue) when some process is writing
writing_process = (None, None)
# We need to buffer updated and deleted objects from the child process to avoid filling up the pipe
updated_objects = []
deleted_objects = []
signal_blocker = SignalBlocker(logger=LOG)
try:
LOG.info('Start polling for executables.')
first_wait = True
sleep_time = 0
while True:
self.registry.backend.query('UNLOCK TABLES')
## Step 4 (easier to do here because we use "continue"s)
if writing_process[1] is not None:
terminated = self.collect_updates(writing_process[1],
updated_objects,
deleted_objects)
if terminated:
writing_process[1].close()
writing_process = (writing_process[0], None)
## Step 5 (easier to do here because we use "continue"s)
completed_processes = self.collect_processes(child_processes)
for proc, status in completed_processes:
if proc is not writing_process[0]:
continue
# drain the queue
if writing_process[1] is not None:
self.collect_updates(writing_process[1],
updated_objects,
deleted_objects,
drain=True)
writing_process[1].close()
writing_process = (None, None)
if status != 'done':
continue
# The child process may send us the list of updated/deleted objects
# Block system signals and get update done
with signal_blocker:
for obj in updated_objects:
self.inventory.update(obj,
write=True,
changelog=CHANGELOG)
for obj in deleted_objects:
CHANGELOG.info('Deleting %s', str(obj))
self.inventory.delete(obj, write=True)
updated_objects = []
deleted_objects = []
## Step 6 (easier to do here because we use "continue"s)
time.sleep(sleep_time)
## Step 1: Poll
LOG.debug('Polling for executables.')
# UNLOCK statement at the top of the while loop
self.registry.backend.query('LOCK TABLES `action` WRITE')
sql = 'SELECT s.`id`, s.`write_request`, s.`title`, s.`path`, s.`args`, s.`user_id`, u.`name`'
sql += ' FROM `action` AS s INNER JOIN `users` AS u ON u.`id` = s.`user_id`'
sql += ' WHERE s.`status` = \'new\''
if writing_process[0] is not None:
# we don't allow write_requesting executables while there is one running
sql += ' AND s.`write_request` = 0'
sql += ' ORDER BY s.`timestamp` LIMIT 1'
result = self.registry.backend.query(sql)
if len(result) == 0:
if len(child_processes) == 0 and first_wait:
LOG.info('Waiting for executables.')
first_wait = False
sleep_time = 0.5
LOG.debug('No executable found, sleeping for %d seconds.',
sleep_time)
continue
## Step 2: If a script is found, check the authorization of the script.
exec_id, write_request, title, path, args, user_id, user_name = result[
0]
first_wait = True
sleep_time = 0
if not os.path.exists(path + '/exec.py'):
LOG.info(
'Executable %s from user %s (write request: %s) not found.',
title, user_name, write_request)
self.registry.backend.query(
'UPDATE `action` SET `status` = %s WHERE `id` = %s',
'notfound', exec_id)
continue
LOG.info(
'Found executable %s from user %s (write request: %s)',
title, user_name, write_request)
proc_args = (path, args)
if write_request:
if not self.check_write_auth(title, user_id, path):
LOG.warning(
'Executable %s from user %s is not authorized for write access.',
title, user_name)
# send a message
self.registry.backend.query(
'UPDATE `action` SET `status` = %s where `id` = %s',
'authfailed', exec_id)
continue
queue = multiprocessing.Queue()
proc_args += (queue, )
## Step 3: Spawn a child process for the script
self.registry.backend.query(
'UPDATE `action` SET `status` = %s WHERE `id` = %s', 'run',
exec_id)
proc = multiprocessing.Process(target=self._run_one,
name=title,
args=proc_args)
child_processes.append((exec_id, proc, user_name, path))
proc.daemon = True
proc.start()
if write_request:
writing_process = (proc, proc_args[-1])
LOG.info('Started executable %s (%s) from user %s (PID %d).',
title, path, user_name, proc.pid)
except KeyboardInterrupt:
LOG.info('Server process was interrupted.')
except:
# log the exception
LOG.warning(
'Exception in server process. Terminating all child processes.'
)
raise
finally:
# If the main process was interrupted by Ctrl+C:
# Ctrl+C will pass SIGINT to all child processes (if this process is the head of the
# foreground process group). In this case calling terminate() will duplicate signals
# in the child. Child processes have to always ignore SIGINT and be killed only from
# SIGTERM sent by the line below.
self.registry.backend.query('UNLOCK TABLES')
for exec_id, proc, user_name, path in child_processes:
LOG.warning('Terminating %s (%s) requested by %s (PID %d)',
proc.name, path, user_name, proc.pid)
uid = os.geteuid()
os.seteuid(0)
proc.terminate()
os.seteuid(uid)
proc.join(5)
if proc.is_alive():
LOG.warning(
'Child process %d did not return after 5 seconds.',
proc.pid)
self.registry.backend.query(
'UPDATE `action` SET `status` = \'killed\' where `id` = %s',
exec_id)
if writing_process[1] is not None:
writing_process[1].close()
def _commit_deletions(self, cycle_number, inventory, deleted, comment):
"""
@param cycle_number Cycle number.
@param inventory Global (original) inventory
@param deleted {dataset_replica: {condition_id: set(block_replicas)}}
@param comment Comment to be passed to the deletion interface.
"""
signal_blocker = SignalBlocker(logger=LOG)
# organize the replicas into sites
deletions_by_site = collections.defaultdict(
list) # {site: [(dataset_replica, block_replicas)]}
for replica, matches in deleted.iteritems():
all_block_replicas = set()
for condition_id, block_replicas in matches.iteritems():
all_block_replicas.update(block_replicas)
deletions_by_site[replica.site].append(
(replica, all_block_replicas))
# now schedule deletions for each site
for site in sorted(deletions_by_site.iterkeys(), key=lambda s: s.name):
site_deletion_list = deletions_by_site[site]
LOG.info('Deleting %d replicas from %s.', len(site_deletion_list),
site.name)
flat_list = []
for replica, block_replicas in site_deletion_list:
if set(block_replicas) == replica.block_replicas:
flat_list.append(replica)
else:
flat_list.extend(block_replicas)
# Block interruptions until deletion is executed and recorded
with signal_blocker:
deletion_mapping = self.deletion_op.schedule_deletions(
flat_list, comments=comment)
total_size = 0
for deletion_id, (approved, site,
items) in deletion_mapping.iteritems():
# Delete ownership of block replicas in the approved deletions.
# Because replicas in partition_repository are modified already during the iterative
# deletion, we find the original replicas from the global inventory.
size = 0
datasets = set()
for item in items:
if type(item) is Dataset:
dataset = inventory.datasets[item.name]
replica = dataset.find_replica(site.name)
for block_replica in replica.block_replicas:
size += block_replica.size
if approved:
block_replica.group = inventory.groups[
None]
inventory.update(block_replica)
else:
dataset = inventory.datasets[item.dataset.name]
block = dataset.find_block(item.name)
replica = block.find_replica(site.name)
if replica is None:
LOG.info('Could not find %s:%s in inventory',
site.name, block.full_name())
raise RuntimeError()
size += replica.size
if approved:
replica.group = inventory.groups[None]
inventory.update(replica)
datasets.add(dataset)
self.history.make_deletion_entry(cycle_number, site,
deletion_id, approved,
datasets, size)
total_size += size
LOG.info('Done deleting %.1f TB from %s.', total_size * 1.e-12,
site.name)