def human_dump(self):
data = super(MoveJob, self).human_dump()
data['src_hostname'] = cache.get_hostname_by_addr(data['src_host'],
strict=False)
data['dst_hostname'] = cache.get_hostname_by_addr(data['dst_host'],
strict=False)
return data
def marker_format(self, marker):
return marker.format(
group_id=str(self.group),
src_host=self.src_host,
src_hostname=cache.get_hostname_by_addr(self.src_host),
src_backend_id=self.src_backend_id,
src_port=str(self.src_port),
src_base_path=self.src_base_path,
dst_host=self.dst_host,
dst_hostname=cache.get_hostname_by_addr(self.dst_host),
dst_port=str(self.dst_port),
dst_base_path=self.dst_base_path,
dst_backend_id=self.dst_backend_id)
def on_exec_stop(self, processor):
hostnames = set([cache.get_hostname_by_addr(host)
for host in (self.host, self.src_host)])
dl = jobs.Job.list(processor.downtimes,
host=list(hostnames), type='network_load')
busy_hostnames = set()
for rec in dl:
if rec['job_id'] != self.parent_job.id:
busy_hostnames.add(rec['host'])
release_hostnames = hostnames - busy_hostnames
if release_hostnames:
try:
for hostname in release_hostnames:
inventory.remove_net_monitoring_downtime(hostname)
except Exception as e:
logger.error('Job {0}, task {1}: failed to remove net monitoring downtime: '
'{2}'.format(self.parent_job.id, self.id, e))
raise
try:
res = processor.downtimes.remove({'job_id': self.parent_job.id,
'host': {'$in': list(hostnames)},
'type': 'network_load'})
if res['ok'] != 1:
raise ValueError('bad response: {0}'.format(res))
except Exception as e:
logger.error('Job {0}, task {1}: unexpected mongo error: '
'{2}'.format(self.parent_job.id, self.id, e))
raise
def on_exec_start(self, processor):
hostnames = set([cache.get_hostname_by_addr(host)
for host in (self.host, self.src_host)])
dl = jobs.Job.list(processor.downtimes,
host=list(hostnames), type='network_load')
set_hostnames = set(record['host'] for record in dl)
not_set_hostnames = hostnames - set_hostnames
if not_set_hostnames:
try:
for hostname in not_set_hostnames:
inventory.set_net_monitoring_downtime(hostname)
except Exception as e:
logger.error('Job {0}, task {1}: failed to set net monitoring downtime: '
'{2}'.format(self.parent_job.id, self.id, e))
raise
try:
bulk_op = processor.downtimes.initialize_unordered_bulk_op()
for hostname in hostnames:
bulk_op.insert({'job_id': self.parent_job.id,
'host': hostname,
'type': 'network_load'})
res = bulk_op.execute()
if res['nInserted'] != len(hostnames):
raise ValueError('failed to set all downtimes: {0}/{1}'.format(
res['nInserted'], len(hostnames)))
except Exception as e:
logger.error('Job {0}, task {1}: unexpected mongo error: '
'{2}'.format(self.parent_job.id, self.id, e))
raise
def info(self):
res = {}
res['node'] = '{0}:{1}:{2}'.format(self.node.host, self.node.port, self.node.family)
res['backend_id'] = self.backend_id
res['addr'] = str(self)
res['hostname'] = cache.get_hostname_by_addr(self.node.host.addr)
res['status'] = self.status
res['status_text'] = self.status_text
res['dc'] = self.node.host.dc
res['last_stat_update'] = (self.stat and
datetime.datetime.fromtimestamp(self.stat.ts).strftime('%Y-%m-%d %H:%M:%S') or
'unknown')
if self.stat:
res['free_space'] = int(self.stat.free_space)
res['effective_space'] = self.effective_space
res['free_effective_space'] = int(max(self.stat.free_space - (self.stat.total_space - self.effective_space), 0))
res['used_space'] = int(self.stat.used_space)
res['total_space'] = int(self.stat.total_space)
res['total_files'] = self.stat.files + self.stat.files_removed
res['records_alive'] = self.stat.files
res['records_removed'] = self.stat.files_removed
res['fragmentation'] = self.stat.fragmentation
res['defrag_state'] = self.stat.defrag_state
if self.base_path:
res['path'] = self.base_path
return res
def __node_in_group(self):
group = self.group in storage.groups and storage.groups[self.group] or None
nb_str = '{0}:{1}/{2}'.format(self.host, self.port, self.backend_id).encode('utf-8')
node_backend = nb_str in storage.node_backends and storage.node_backends[nb_str] or None
if group and node_backend:
logger.debug('Job {0}, task {1}: checking node backend {2} '
'with group {3} node backends: {4}'.format(
self.parent_job.id, self.id, node_backend, self.group, group.node_backends))
nb_in_group = node_backend.group is group
else:
nb_in_group = False
try:
hostname = cache.get_hostname_by_addr(self.host)
except CacheUpstreamError:
raise ValueError('Failed to resolve job host {}'.format(self.host))
nb_in_history = infrastructure.node_backend_in_last_history_state(
self.group, hostname, self.port, self.backend_id)
logger.debug('Job {0}, task {1}: checking node backend {2} '
'in group {3} history set: {4}'.format(
self.parent_job.id, self.id, nb_str, self.group, nb_in_history))
if nb_in_group:
logger.info('Job {0}, task {1}: node backend {2} is still '
'in group {3}'.format(self.parent_job.id, self.id, nb_str, self.group))
if nb_in_history:
logger.info('Job {0}, task {1}: node backend {2} is still '
'in group\'s {3} history'.format(
self.parent_job.id, self.id, nb_str, self.group))
return nb_in_group or nb_in_history
def __hostnames(self, hosts):
hostnames = []
for host in hosts:
try:
hostnames.append(cache.get_hostname_by_addr(host))
except CacheUpstreamError:
raise RuntimeError('Failed to resolve host {0}'.format(host))
return hostnames
def __hostnames(self, hosts):
hostnames = []
for host in hosts:
try:
hostnames.append(cache.get_hostname_by_addr(host))
except CacheUpstreamError:
raise RuntimeError('Failed to resolve host {0}'.format(host))
return hostnames
def execute(self):
try:
hostname = cache.get_hostname_by_addr(self.host)
except CacheUpstreamError:
raise ValueError('Failed to resolve job host {}'.format(self.host))
nb_hostname_str = '{0}:{1}/{2}'.format(hostname, self.port,
self.backend_id).encode('utf-8')
try:
logger.info('Job {0}, task {1}: removing node backend {2} '
'from group {3} history'.format(
self.parent_job.id, self.id, nb_hostname_str,
self.group))
infrastructure.detach_node(
group_id=self.group,
hostname=hostname,
port=self.port,
family=self.family,
backend_id=self.backend_id,
record_type=history.GroupStateRecord.HISTORY_RECORD_JOB,
)
logger.info('Job {0}, task {1}: removed node backend {2} '
'from group {3} history'.format(
self.parent_job.id, self.id, nb_hostname_str,
self.group))
except ValueError as e:
# TODO: Think about changing ValueError to some dedicated exception
# to differentiate between event when there is no such node in group
# and an actual ValueError being raised
logger.error(
'Job {0}, task {1}: failed to remove node backend {2} '
'from group {3} history: {4}'.format(self.parent_job.id,
self.id, nb_hostname_str,
self.group, e))
pass
group = self.group in storage.groups and storage.groups[
self.group] or None
nb_str = '{0}:{1}/{2}'.format(self.host, self.port,
self.backend_id).encode('utf-8')
node_backend = nb_str in storage.node_backends and storage.node_backends[
nb_str] or None
if group and node_backend and node_backend in group.node_backends:
logger.info('Job {0}, task {1}: removing node backend {2} '
'from group {3} node backends'.format(
self.parent_job.id, self.id, node_backend, group))
group.remove_node_backend(node_backend)
group.update_status_recursive()
logger.info('Job {0}, task {1}: removed node backend {2} '
'from group {3} node backends'.format(
self.parent_job.id, self.id, node_backend, group))
def execute(self):
try:
hostname = cache.get_hostname_by_addr(self.host)
except CacheUpstreamError:
raise ValueError('Failed to resolve job host {}'.format(self.host))
nb_hostname_str = '{0}:{1}/{2}'.format(
hostname, self.port, self.backend_id
).encode('utf-8')
try:
logger.info('Job {0}, task {1}: removing node backend {2} '
'from group {3} history'.format(
self.parent_job.id, self.id, nb_hostname_str, self.group))
infrastructure.detach_node(
group_id=self.group,
hostname=hostname,
port=self.port,
family=self.family,
backend_id=self.backend_id,
record_type=history.GroupStateRecord.HISTORY_RECORD_JOB,
)
logger.info('Job {0}, task {1}: removed node backend {2} '
'from group {3} history'.format(
self.parent_job.id, self.id, nb_hostname_str, self.group))
except ValueError as e:
# TODO: Think about changing ValueError to some dedicated exception
# to differentiate between event when there is no such node in group
# and an actual ValueError being raised
logger.error('Job {0}, task {1}: failed to remove node backend {2} '
'from group {3} history: {4}'.format(
self.parent_job.id, self.id, nb_hostname_str, self.group, e))
pass
group = self.group in storage.groups and storage.groups[self.group] or None
nb_str = '{0}:{1}/{2}'.format(self.host, self.port, self.backend_id).encode('utf-8')
node_backend = nb_str in storage.node_backends and storage.node_backends[nb_str] or None
if group and node_backend and node_backend in group.node_backends:
logger.info('Job {0}, task {1}: removing node backend {2} '
'from group {3} node backends'.format(
self.parent_job.id, self.id, node_backend, group))
group.remove_node_backend(node_backend)
group.update_status_recursive()
logger.info('Job {0}, task {1}: removed node backend {2} '
'from group {3} node backends'.format(
self.parent_job.id, self.id, node_backend, group))
def marker_format(self, marker):
hostnames = []
for host in (self.src_host, self.dst_host):
try:
hostnames.append(cache.get_hostname_by_addr(host))
except CacheUpstreamError:
raise RuntimeError('Failed to resolve host {0}'.format(host))
src_hostname, dst_hostname = hostnames
return marker.format(group_id=str(self.group),
src_host=self.src_host,
src_hostname=src_hostname,
src_backend_id=self.src_backend_id,
src_port=str(self.src_port),
src_base_path=self.src_base_path,
dst_host=self.dst_host,
dst_hostname=dst_hostname,
dst_port=str(self.dst_port),
dst_base_path=self.dst_base_path,
dst_backend_id=self.dst_backend_id)
def marker_format(self, marker):
hostnames = []
for host in (self.src_host, self.dst_host):
try:
hostnames.append(cache.get_hostname_by_addr(host))
except CacheUpstreamError:
raise RuntimeError('Failed to resolve host {0}'.format(host))
src_hostname, dst_hostname = hostnames
return marker.format(
group_id=str(self.group),
src_host=self.src_host,
src_hostname=src_hostname,
src_backend_id=self.src_backend_id,
src_port=str(self.src_port),
src_base_path=self.src_base_path,
dst_host=self.dst_host,
dst_hostname=dst_hostname,
dst_port=str(self.dst_port),
dst_base_path=self.dst_base_path,
dst_backend_id=self.dst_backend_id)
def __node_in_group(self):
group = self.group in storage.groups and storage.groups[
self.group] or None
nb_str = '{0}:{1}/{2}'.format(self.host, self.port,
self.backend_id).encode('utf-8')
node_backend = nb_str in storage.node_backends and storage.node_backends[
nb_str] or None
if group and node_backend:
logger.debug('Job {0}, task {1}: checking node backend {2} '
'with group {3} node backends: {4}'.format(
self.parent_job.id, self.id, node_backend,
self.group, group.node_backends))
nb_in_group = node_backend.group is group
else:
nb_in_group = False
try:
hostname = cache.get_hostname_by_addr(self.host)
except CacheUpstreamError:
raise ValueError('Failed to resolve job host {}'.format(self.host))
nb_in_history = infrastructure.node_backend_in_last_history_state(
self.group, hostname, self.port, self.backend_id)
logger.debug('Job {0}, task {1}: checking node backend {2} '
'in group {3} history set: {4}'.format(
self.parent_job.id, self.id, nb_str, self.group,
nb_in_history))
if nb_in_group:
logger.info('Job {0}, task {1}: node backend {2} is still '
'in group {3}'.format(self.parent_job.id, self.id,
nb_str, self.group))
if nb_in_history:
logger.info('Job {0}, task {1}: node backend {2} is still '
'in group\'s {3} history'.format(
self.parent_job.id, self.id, nb_str, self.group))
return nb_in_group or nb_in_history
def create_tasks(self):
group = storage.groups[self.group]
src_group = storage.groups[self.src_group]
self.check_node_backends(src_group)
old_group_state = infrastructure.get_group_history(group.group_id)['nodes'][-1]['set']
if len(old_group_state) != 1:
raise JobBrokenError('History of group {0} lists {1} '
'node backends, 1 expected'.format(self.group, len(old_group_state)))
old_host, old_port, old_family, old_backend_id, old_base_path = old_group_state[0][:5]
if self.uncoupled_group:
uncoupled_group = storage.groups[self.uncoupled_group]
self.check_node_backends(uncoupled_group)
dst_host = uncoupled_group.node_backends[0].node.host.addr
dst_port = uncoupled_group.node_backends[0].node.port
dst_family = uncoupled_group.node_backends[0].node.family
dst_base_path = uncoupled_group.node_backends[0].base_path
dst_backend_id = uncoupled_group.node_backends[0].backend_id
else:
# gettings dst_* from group history
dst_host, dst_port, dst_backend_id = old_host, old_port, old_backend_id
# TODO: Fix hardcoded family value
dst_base_path, dst_family = old_base_path, old_family
dst_node_backend = self.node_backend(dst_host, dst_port, dst_backend_id)
group_file = (os.path.join(dst_base_path,
self.GROUP_FILE_PATH)
if self.GROUP_FILE_PATH else
'')
remove_path = ''
if self.GROUP_FILE_DIR_MOVE_DST_RENAME and group_file:
remove_path = os.path.join(
dst_base_path, self.GROUP_FILE_DIR_MOVE_DST_RENAME)
if self.uncoupled_group:
for group_id in self.merged_groups:
merged_group = storage.groups[group_id]
merged_nb = merged_group.node_backends[0]
merged_group_file = (os.path.join(merged_nb.base_path,
self.GROUP_FILE_PATH)
if self.GROUP_FILE_PATH else
'')
merged_path = ''
if self.MERGE_GROUP_FILE_DIR_MOVE_SRC_RENAME and merged_group_file:
merged_path = os.path.join(
merged_nb.base_path, self.MERGE_GROUP_FILE_DIR_MOVE_SRC_RENAME)
node_backend_str = self.node_backend(merged_nb.node.host.addr,
merged_nb.node.port,
merged_nb.backend_id)
merged_group_file_marker = (os.path.join(merged_nb.base_path,
self.MERGE_GROUP_FILE_MARKER_PATH)
if self.MERGE_GROUP_FILE_MARKER_PATH else
'')
shutdown_cmd = infrastructure._disable_node_backend_cmd(
merged_nb.node.host.addr,
merged_nb.node.port,
merged_nb.node.family,
merged_nb.backend_id)
params = {'node_backend': node_backend_str.encode('utf-8'),
'group': str(group_id),
'merged_to': str(self.uncoupled_group),
'remove_group_file': merged_group_file}
if merged_group_file_marker:
params['group_file_marker'] = merged_group_file_marker.format(
dst_group_id=self.uncoupled_group,
dst_backend_id=dst_backend_id)
if merged_path:
params['move_src'] = os.path.dirname(merged_group_file)
params['move_dst'] = merged_path
task = NodeStopTask.new(self,
group=group_id,
uncoupled=True,
host=merged_nb.node.host.addr,
cmd=shutdown_cmd,
params=params)
self.tasks.append(task)
reconfigure_cmd = infrastructure._reconfigure_node_cmd(
merged_nb.node.host.addr,
merged_nb.node.port,
merged_nb.node.family)
task = MinionCmdTask.new(self,
host=merged_nb.node.host.addr,
cmd=reconfigure_cmd,
params={'node_backend': node_backend_str.encode('utf-8')})
self.tasks.append(task)
task = HistoryRemoveNodeTask.new(self,
group=group_id,
host=merged_nb.node.host.addr,
port=merged_nb.node.port,
backend_id=merged_nb.backend_id)
self.tasks.append(task)
# destination backend should be stopped
shutdown_cmd = infrastructure._disable_node_backend_cmd(
dst_host, dst_port, dst_family, dst_backend_id)
params = {'node_backend': dst_node_backend.encode('utf-8'),
'group': str(self.uncoupled_group)}
if remove_path:
params['move_src'] = os.path.join(os.path.dirname(group_file))
params['move_dst'] = remove_path
task = NodeStopTask.new(self,
group=self.uncoupled_group,
uncoupled=True,
host=dst_host,
cmd=shutdown_cmd,
params=params)
self.tasks.append(task)
src_backend = src_group.node_backends[0]
restore_backend = group.node_backends and group.node_backends[0]
make_readonly = (src_backend is not restore_backend and
src_backend.status == storage.Status.OK)
mark_src_backend = self.make_path(
self.BACKEND_DOWN_MARKER, base_path=src_backend.base_path).format(
backend_id=src_backend.backend_id)
if make_readonly:
make_readonly_cmd = infrastructure._make_readonly_node_backend_cmd(
src_backend.node.host.addr, src_backend.node.port,
src_backend.node.family, src_backend.backend_id)
task = MinionCmdTask.new(self,
host=src_backend.node.host.addr,
cmd=make_readonly_cmd,
params={'node_backend': self.node_backend(
src_backend.node.host.addr, src_backend.node.port,
src_backend.backend_id).encode('utf-8'),
'mark_backend': mark_src_backend})
self.tasks.append(task)
move_cmd = infrastructure.move_group_cmd(
src_host=src_group.node_backends[0].node.host.addr,
src_path=src_group.node_backends[0].base_path,
src_family=src_group.node_backends[0].node.family,
dst_path=dst_base_path)
params = {'group': str(self.group),
'group_file': group_file}
if remove_path and self.uncoupled_group:
params['remove_path'] = remove_path
task = RsyncBackendTask.new(self,
host=dst_host,
src_host=old_host,
group=self.group,
cmd=move_cmd,
node_backend=self.node_backend(old_host, old_port, old_backend_id),
params=params)
self.tasks.append(task)
additional_files = config.get('restore', {}).get('restore_additional_files', [])
for src_file_tpl, dst_file_path in additional_files:
rsync_cmd = infrastructure.move_group_cmd(
src_host=src_group.node_backends[0].node.host.addr,
src_path=src_group.node_backends[0].base_path,
src_family=src_group.node_backends[0].node.family,
dst_path=os.path.join(dst_base_path, dst_file_path),
file_tpl=src_file_tpl)
params = {'group': str(self.group)}
task = MinionCmdTask.new(self,
host=dst_host,
group=self.group,
cmd=rsync_cmd,
params=params)
self.tasks.append(task)
if (restore_backend and restore_backend.status in
storage.NodeBackend.ACTIVE_STATUSES):
nb = group.node_backends[0]
shutdown_cmd = infrastructure._disable_node_backend_cmd(
nb.node.host.addr, nb.node.port, nb.node.family, nb.backend_id)
group_file = (os.path.join(nb.base_path,
self.GROUP_FILE_PATH)
if self.GROUP_FILE_PATH else
'')
group_file_marker = (os.path.join(nb.base_path,
self.GROUP_FILE_MARKER_PATH)
if self.GROUP_FILE_MARKER_PATH else
'')
group_file_marker_fmt = group_file_marker.format(
group_id=str(self.group),
src_host=nb.node.host.addr,
src_hostname=cache.get_hostname_by_addr(
nb.node.host.addr),
src_backend_id=nb.backend_id,
src_port=str(nb.node.port),
src_base_path=nb.base_path,
dst_host=dst_host,
dst_hostname=cache.get_hostname_by_addr(dst_host),
dst_port=str(dst_port),
dst_base_path=dst_base_path,
dst_backend_id=dst_backend_id)
params = {'node_backend': str(nb).encode('utf-8'),
'group': str(self.group),
'group_file_marker': group_file_marker_fmt,
'remove_group_file': group_file}
if self.GROUP_FILE_DIR_MOVE_SRC_RENAME and group_file:
params['move_src'] = os.path.join(os.path.dirname(group_file))
params['move_dst'] = os.path.join(
nb.base_path, self.GROUP_FILE_DIR_MOVE_SRC_RENAME)
task = NodeStopTask.new(self,
group=self.group,
host=nb.node.host.addr,
cmd=shutdown_cmd,
params=params)
self.tasks.append(task)
reconfigure_cmd = infrastructure._reconfigure_node_cmd(
dst_host, dst_port, dst_family)
task = MinionCmdTask.new(self,
host=dst_host,
cmd=reconfigure_cmd,
params={'node_backend': dst_node_backend.encode('utf-8')})
self.tasks.append(task)
if self.uncoupled_group:
task = HistoryRemoveNodeTask.new(self,
group=self.group,
host=old_host,
port=old_port,
backend_id=old_backend_id)
self.tasks.append(task)
task = HistoryRemoveNodeTask.new(self,
group=self.uncoupled_group,
host=dst_host,
port=dst_port,
backend_id=dst_backend_id)
self.tasks.append(task)
start_cmd = infrastructure._enable_node_backend_cmd(
dst_host, dst_port, dst_family, dst_backend_id)
task = MinionCmdTask.new(self,
host=dst_host,
cmd=start_cmd,
params={'node_backend': dst_node_backend.encode('utf-8')})
self.tasks.append(task)
if make_readonly:
make_writable_cmd = infrastructure._make_writable_node_backend_cmd(
src_backend.node.host.addr, src_backend.node.port,
src_backend.node.family, src_backend.backend_id)
task = MinionCmdTask.new(self,
host=src_backend.node.host.addr,
cmd=make_writable_cmd,
params={'node_backend': self.node_backend(
src_backend.node.host.addr, src_backend.node.port,
src_backend.backend_id).encode('utf-8'),
'unmark_backend': mark_src_backend,
'success_codes': [self.DNET_CLIENT_ALREADY_IN_PROGRESS]})
self.tasks.append(task)
# This start command is executed for the case when elliptics
# was restarted between make_readonly and make_writable commands
start_cmd = infrastructure._enable_node_backend_cmd(
src_backend.node.host.addr,
src_backend.node.port,
src_backend.node.family,
src_backend.backend_id)
task = MinionCmdTask.new(self,
host=src_backend.node.host.addr,
cmd=start_cmd,
params={'node_backend': str(src_backend).encode('utf-8'),
'success_codes': [self.DNET_CLIENT_ALREADY_IN_PROGRESS]})
self.tasks.append(task)
def human_dump(self):
data = super(HistoryRemoveNodeTask, self).human_dump()
data['hostname'] = cache.get_hostname_by_addr(data['host'],
strict=False)
return data
def human_dump(self):
data = super(RecoverDcJob, self).human_dump()
data['hostname'] = cache.get_hostname_by_addr(data['host'], strict=False)
return data
def create_tasks(self):
group = storage.groups[self.group]
src_group = storage.groups[self.src_group]
self.check_node_backends(src_group)
group_history = infrastructure.get_group_history(group.group_id)
old_group_node_backends_set = group_history.nodes and group_history.nodes[-1].set or []
if old_group_node_backends_set:
old_node_backend = old_group_node_backends_set[0]
old_host = cache.get_ip_address_by_host(old_node_backend.hostname)
old_port = old_node_backend.port
old_family = old_node_backend.family
old_backend_id = old_node_backend.backend_id
old_base_path = old_node_backend.path
if self.uncoupled_group:
uncoupled_group = storage.groups[self.uncoupled_group]
self.check_node_backends(uncoupled_group)
dst_host = uncoupled_group.node_backends[0].node.host.addr
dst_port = uncoupled_group.node_backends[0].node.port
dst_family = uncoupled_group.node_backends[0].node.family
dst_base_path = uncoupled_group.node_backends[0].base_path
dst_backend_id = uncoupled_group.node_backends[0].backend_id
else:
if len(old_group_node_backends_set) != 1:
raise JobBrokenError(
'History of group {0} lists {1} node backends, 1 expected'.format(
self.group, len(old_group_node_backends_set)
)
)
# gettings dst_* from group history
dst_host, dst_port, dst_backend_id = old_host, old_port, old_backend_id
# TODO: Fix hardcoded family value
dst_base_path, dst_family = old_base_path, old_family
dst_node_backend = self.node_backend(dst_host, dst_port, dst_backend_id)
group_file = (os.path.join(dst_base_path, self.GROUP_FILE_PATH)
if self.GROUP_FILE_PATH else
'')
remove_path = ''
if self.GROUP_FILE_DIR_MOVE_DST_RENAME and group_file:
remove_path = os.path.join(
dst_base_path, self.GROUP_FILE_DIR_MOVE_DST_RENAME)
if self.uncoupled_group:
for group_id in self.merged_groups:
merged_group = storage.groups[group_id]
merged_nb = merged_group.node_backends[0]
merged_group_file = (os.path.join(merged_nb.base_path,
self.GROUP_FILE_PATH)
if self.GROUP_FILE_PATH else
'')
merged_path = ''
if self.MERGE_GROUP_FILE_DIR_MOVE_SRC_RENAME and merged_group_file:
merged_path = os.path.join(
merged_nb.base_path, self.MERGE_GROUP_FILE_DIR_MOVE_SRC_RENAME)
node_backend_str = self.node_backend(merged_nb.node.host.addr,
merged_nb.node.port,
merged_nb.backend_id)
merged_group_file_marker = (os.path.join(merged_nb.base_path,
self.MERGE_GROUP_FILE_MARKER_PATH)
if self.MERGE_GROUP_FILE_MARKER_PATH else
'')
shutdown_cmd = infrastructure._disable_node_backend_cmd(
merged_nb.node.host.addr,
merged_nb.node.port,
merged_nb.node.family,
merged_nb.backend_id)
params = {'node_backend': node_backend_str.encode('utf-8'),
'group': str(group_id),
'merged_to': str(self.uncoupled_group),
'remove_group_file': merged_group_file}
if merged_group_file_marker:
params['group_file_marker'] = merged_group_file_marker.format(
dst_group_id=self.uncoupled_group,
dst_backend_id=dst_backend_id)
if merged_path:
params['move_src'] = os.path.dirname(merged_group_file)
params['move_dst'] = merged_path
task = NodeStopTask.new(self,
group=group_id,
uncoupled=True,
host=merged_nb.node.host.addr,
cmd=shutdown_cmd,
params=params)
self.tasks.append(task)
reconfigure_cmd = infrastructure._reconfigure_node_cmd(
merged_nb.node.host.addr,
merged_nb.node.port,
merged_nb.node.family)
task = MinionCmdTask.new(self,
host=merged_nb.node.host.addr,
cmd=reconfigure_cmd,
params={'node_backend': node_backend_str.encode('utf-8')})
self.tasks.append(task)
task = HistoryRemoveNodeTask.new(
self,
group=group_id,
host=merged_nb.node.host.addr,
port=merged_nb.node.port,
family=merged_nb.node.family,
backend_id=merged_nb.backend_id,
)
self.tasks.append(task)
# destination backend should be stopped
shutdown_cmd = infrastructure._disable_node_backend_cmd(
dst_host, dst_port, dst_family, dst_backend_id)
params = {'node_backend': dst_node_backend.encode('utf-8'),
'group': str(self.uncoupled_group)}
if remove_path:
params['move_src'] = os.path.join(os.path.dirname(group_file))
params['move_dst'] = remove_path
task = NodeStopTask.new(self,
group=self.uncoupled_group,
uncoupled=True,
host=dst_host,
cmd=shutdown_cmd,
params=params)
self.tasks.append(task)
src_backend = src_group.node_backends[0]
restore_backend = group.node_backends and group.node_backends[0]
make_readonly = (src_backend is not restore_backend and
src_backend.status == storage.Status.OK)
mark_src_backend = self.make_path(
self.BACKEND_DOWN_MARKER, base_path=src_backend.base_path).format(
backend_id=src_backend.backend_id)
if make_readonly:
make_readonly_cmd = infrastructure._make_readonly_node_backend_cmd(
src_backend.node.host.addr, src_backend.node.port,
src_backend.node.family, src_backend.backend_id)
task = MinionCmdTask.new(self,
host=src_backend.node.host.addr,
cmd=make_readonly_cmd,
params={'node_backend': self.node_backend(
src_backend.node.host.addr, src_backend.node.port,
src_backend.backend_id).encode('utf-8'),
'mark_backend': mark_src_backend})
self.tasks.append(task)
move_cmd = infrastructure.move_group_cmd(
src_host=src_group.node_backends[0].node.host.addr,
src_path=src_group.node_backends[0].base_path,
src_family=src_group.node_backends[0].node.family,
dst_path=dst_base_path)
ids_file = (os.path.join(dst_base_path, self.IDS_FILE_PATH)
if self.IDS_FILE_PATH else
'')
params = {'group': str(self.group),
'group_file': group_file,
'ids': ids_file}
if remove_path and self.uncoupled_group:
params['remove_path'] = remove_path
check_node_backend = None
if old_group_node_backends_set:
check_node_backend = self.node_backend(old_host, old_port, old_backend_id)
task = RsyncBackendTask.new(
self,
host=dst_host,
src_host=src_group.node_backends[0].node.host.addr,
group=self.group,
cmd=move_cmd,
node_backend=check_node_backend,
params=params
)
self.tasks.append(task)
additional_files = config.get('restore', {}).get('restore_additional_files', [])
for src_file_tpl, dst_file_path in additional_files:
rsync_cmd = infrastructure.move_group_cmd(
src_host=src_group.node_backends[0].node.host.addr,
src_path=src_group.node_backends[0].base_path,
src_family=src_group.node_backends[0].node.family,
dst_path=os.path.join(dst_base_path, dst_file_path),
file_tpl=src_file_tpl)
params = {'group': str(self.group)}
task = MinionCmdTask.new(self,
host=dst_host,
group=self.group,
cmd=rsync_cmd,
params=params)
self.tasks.append(task)
if (restore_backend and restore_backend.status in
storage.NodeBackend.ACTIVE_STATUSES):
nb = group.node_backends[0]
shutdown_cmd = infrastructure._disable_node_backend_cmd(
nb.node.host.addr, nb.node.port, nb.node.family, nb.backend_id)
group_file = (os.path.join(nb.base_path,
self.GROUP_FILE_PATH)
if self.GROUP_FILE_PATH else
'')
group_file_marker = (os.path.join(nb.base_path,
self.GROUP_FILE_MARKER_PATH)
if self.GROUP_FILE_MARKER_PATH else
'')
hostnames = []
for host in (nb.node.host.addr, dst_host):
try:
hostnames.append(cache.get_hostname_by_addr(host))
except CacheUpstreamError:
raise RuntimeError('Failed to resolve host {0}'.format(host))
src_hostname, dst_hostname = hostnames
group_file_marker_fmt = group_file_marker.format(
group_id=str(self.group),
src_host=nb.node.host.addr,
src_hostname=src_hostname,
src_backend_id=nb.backend_id,
src_port=str(nb.node.port),
src_base_path=nb.base_path,
dst_host=dst_host,
dst_hostname=dst_hostname,
dst_port=str(dst_port),
dst_base_path=dst_base_path,
dst_backend_id=dst_backend_id)
params = {'node_backend': str(nb).encode('utf-8'),
'group': str(self.group),
'group_file_marker': group_file_marker_fmt,
'remove_group_file': group_file}
if self.GROUP_FILE_DIR_MOVE_SRC_RENAME and group_file:
params['move_src'] = os.path.join(os.path.dirname(group_file))
params['move_dst'] = os.path.join(
nb.base_path, self.GROUP_FILE_DIR_MOVE_SRC_RENAME)
task = NodeStopTask.new(self,
group=self.group,
host=nb.node.host.addr,
cmd=shutdown_cmd,
params=params)
self.tasks.append(task)
reconfigure_cmd = infrastructure._reconfigure_node_cmd(
dst_host, dst_port, dst_family)
task = MinionCmdTask.new(self,
host=dst_host,
cmd=reconfigure_cmd,
params={'node_backend': dst_node_backend.encode('utf-8')})
self.tasks.append(task)
if self.uncoupled_group:
if old_group_node_backends_set:
task = HistoryRemoveNodeTask.new(
self,
group=self.group,
host=old_host,
port=old_port,
family=old_family,
backend_id=old_backend_id,
)
self.tasks.append(task)
task = HistoryRemoveNodeTask.new(
self,
group=self.uncoupled_group,
host=dst_host,
port=dst_port,
family=dst_family,
backend_id=dst_backend_id,
)
self.tasks.append(task)
start_cmd = infrastructure._enable_node_backend_cmd(
dst_host, dst_port, dst_family, dst_backend_id)
task = MinionCmdTask.new(self,
host=dst_host,
cmd=start_cmd,
params={'node_backend': dst_node_backend.encode('utf-8')})
self.tasks.append(task)
if make_readonly:
make_writable_cmd = infrastructure._make_writable_node_backend_cmd(
src_backend.node.host.addr, src_backend.node.port,
src_backend.node.family, src_backend.backend_id)
task = MinionCmdTask.new(self,
host=src_backend.node.host.addr,
cmd=make_writable_cmd,
params={'node_backend': self.node_backend(
src_backend.node.host.addr, src_backend.node.port,
src_backend.backend_id).encode('utf-8'),
'unmark_backend': mark_src_backend,
'success_codes': [self.DNET_CLIENT_ALREADY_IN_PROGRESS]})
self.tasks.append(task)
# This start command is executed for the case when elliptics
# was restarted between make_readonly and make_writable commands
start_cmd = infrastructure._enable_node_backend_cmd(
src_backend.node.host.addr,
src_backend.node.port,
src_backend.node.family,
src_backend.backend_id)
task = MinionCmdTask.new(self,
host=src_backend.node.host.addr,
cmd=start_cmd,
params={'node_backend': str(src_backend).encode('utf-8'),
'success_codes': [self.DNET_CLIENT_ALREADY_IN_PROGRESS]})
self.tasks.append(task)
def human_dump(self):
data = super(HistoryRemoveNodeTask, self).human_dump()
data['hostname'] = cache.get_hostname_by_addr(data['host'], strict=False)
return data
def parents(self):
return cache.get_host_tree(
cache.get_hostname_by_addr(self.addr))
def human_dump(self):
data = super(MinionCmdTask, self).human_dump()
data['hostname'] = cache.get_hostname_by_addr(data['host'], strict=False)
return data
def human_dump(self):
data = super(MoveJob, self).human_dump()
data['src_hostname'] = cache.get_hostname_by_addr(data['src_host'], strict=False)
data['dst_hostname'] = cache.get_hostname_by_addr(data['dst_host'], strict=False)
return data
def human_dump(self):
data = super(MinionCmdTask, self).human_dump()
data['hostname'] = cache.get_hostname_by_addr(data['host'],
strict=False)
return data
def human_dump(self):
data = super(RecoverDcJob, self).human_dump()
data['hostname'] = cache.get_hostname_by_addr(data['host'])
return data
def hostname(self):
return cache.get_hostname_by_addr(self.addr)
