def human_dump(self): data = super(MoveJob, self).human_dump() data['src_hostname'] = cache.get_hostname_by_addr(data['src_host'], strict=False) data['dst_hostname'] = cache.get_hostname_by_addr(data['dst_host'], strict=False) return data
def marker_format(self, marker): return marker.format( group_id=str(self.group), src_host=self.src_host, src_hostname=cache.get_hostname_by_addr(self.src_host), src_backend_id=self.src_backend_id, src_port=str(self.src_port), src_base_path=self.src_base_path, dst_host=self.dst_host, dst_hostname=cache.get_hostname_by_addr(self.dst_host), dst_port=str(self.dst_port), dst_base_path=self.dst_base_path, dst_backend_id=self.dst_backend_id)
def on_exec_stop(self, processor): hostnames = set([cache.get_hostname_by_addr(host) for host in (self.host, self.src_host)]) dl = jobs.Job.list(processor.downtimes, host=list(hostnames), type='network_load') busy_hostnames = set() for rec in dl: if rec['job_id'] != self.parent_job.id: busy_hostnames.add(rec['host']) release_hostnames = hostnames - busy_hostnames if release_hostnames: try: for hostname in release_hostnames: inventory.remove_net_monitoring_downtime(hostname) except Exception as e: logger.error('Job {0}, task {1}: failed to remove net monitoring downtime: ' '{2}'.format(self.parent_job.id, self.id, e)) raise try: res = processor.downtimes.remove({'job_id': self.parent_job.id, 'host': {'$in': list(hostnames)}, 'type': 'network_load'}) if res['ok'] != 1: raise ValueError('bad response: {0}'.format(res)) except Exception as e: logger.error('Job {0}, task {1}: unexpected mongo error: ' '{2}'.format(self.parent_job.id, self.id, e)) raise
def on_exec_start(self, processor): hostnames = set([cache.get_hostname_by_addr(host) for host in (self.host, self.src_host)]) dl = jobs.Job.list(processor.downtimes, host=list(hostnames), type='network_load') set_hostnames = set(record['host'] for record in dl) not_set_hostnames = hostnames - set_hostnames if not_set_hostnames: try: for hostname in not_set_hostnames: inventory.set_net_monitoring_downtime(hostname) except Exception as e: logger.error('Job {0}, task {1}: failed to set net monitoring downtime: ' '{2}'.format(self.parent_job.id, self.id, e)) raise try: bulk_op = processor.downtimes.initialize_unordered_bulk_op() for hostname in hostnames: bulk_op.insert({'job_id': self.parent_job.id, 'host': hostname, 'type': 'network_load'}) res = bulk_op.execute() if res['nInserted'] != len(hostnames): raise ValueError('failed to set all downtimes: {0}/{1}'.format( res['nInserted'], len(hostnames))) except Exception as e: logger.error('Job {0}, task {1}: unexpected mongo error: ' '{2}'.format(self.parent_job.id, self.id, e)) raise
def info(self): res = {} res['node'] = '{0}:{1}:{2}'.format(self.node.host, self.node.port, self.node.family) res['backend_id'] = self.backend_id res['addr'] = str(self) res['hostname'] = cache.get_hostname_by_addr(self.node.host.addr) res['status'] = self.status res['status_text'] = self.status_text res['dc'] = self.node.host.dc res['last_stat_update'] = (self.stat and datetime.datetime.fromtimestamp(self.stat.ts).strftime('%Y-%m-%d %H:%M:%S') or 'unknown') if self.stat: res['free_space'] = int(self.stat.free_space) res['effective_space'] = self.effective_space res['free_effective_space'] = int(max(self.stat.free_space - (self.stat.total_space - self.effective_space), 0)) res['used_space'] = int(self.stat.used_space) res['total_space'] = int(self.stat.total_space) res['total_files'] = self.stat.files + self.stat.files_removed res['records_alive'] = self.stat.files res['records_removed'] = self.stat.files_removed res['fragmentation'] = self.stat.fragmentation res['defrag_state'] = self.stat.defrag_state if self.base_path: res['path'] = self.base_path return res
def __node_in_group(self): group = self.group in storage.groups and storage.groups[self.group] or None nb_str = '{0}:{1}/{2}'.format(self.host, self.port, self.backend_id).encode('utf-8') node_backend = nb_str in storage.node_backends and storage.node_backends[nb_str] or None if group and node_backend: logger.debug('Job {0}, task {1}: checking node backend {2} ' 'with group {3} node backends: {4}'.format( self.parent_job.id, self.id, node_backend, self.group, group.node_backends)) nb_in_group = node_backend.group is group else: nb_in_group = False try: hostname = cache.get_hostname_by_addr(self.host) except CacheUpstreamError: raise ValueError('Failed to resolve job host {}'.format(self.host)) nb_in_history = infrastructure.node_backend_in_last_history_state( self.group, hostname, self.port, self.backend_id) logger.debug('Job {0}, task {1}: checking node backend {2} ' 'in group {3} history set: {4}'.format( self.parent_job.id, self.id, nb_str, self.group, nb_in_history)) if nb_in_group: logger.info('Job {0}, task {1}: node backend {2} is still ' 'in group {3}'.format(self.parent_job.id, self.id, nb_str, self.group)) if nb_in_history: logger.info('Job {0}, task {1}: node backend {2} is still ' 'in group\'s {3} history'.format( self.parent_job.id, self.id, nb_str, self.group)) return nb_in_group or nb_in_history
def __hostnames(self, hosts): hostnames = [] for host in hosts: try: hostnames.append(cache.get_hostname_by_addr(host)) except CacheUpstreamError: raise RuntimeError('Failed to resolve host {0}'.format(host)) return hostnames
def execute(self): try: hostname = cache.get_hostname_by_addr(self.host) except CacheUpstreamError: raise ValueError('Failed to resolve job host {}'.format(self.host)) nb_hostname_str = '{0}:{1}/{2}'.format(hostname, self.port, self.backend_id).encode('utf-8') try: logger.info('Job {0}, task {1}: removing node backend {2} ' 'from group {3} history'.format( self.parent_job.id, self.id, nb_hostname_str, self.group)) infrastructure.detach_node( group_id=self.group, hostname=hostname, port=self.port, family=self.family, backend_id=self.backend_id, record_type=history.GroupStateRecord.HISTORY_RECORD_JOB, ) logger.info('Job {0}, task {1}: removed node backend {2} ' 'from group {3} history'.format( self.parent_job.id, self.id, nb_hostname_str, self.group)) except ValueError as e: # TODO: Think about changing ValueError to some dedicated exception # to differentiate between event when there is no such node in group # and an actual ValueError being raised logger.error( 'Job {0}, task {1}: failed to remove node backend {2} ' 'from group {3} history: {4}'.format(self.parent_job.id, self.id, nb_hostname_str, self.group, e)) pass group = self.group in storage.groups and storage.groups[ self.group] or None nb_str = '{0}:{1}/{2}'.format(self.host, self.port, self.backend_id).encode('utf-8') node_backend = nb_str in storage.node_backends and storage.node_backends[ nb_str] or None if group and node_backend and node_backend in group.node_backends: logger.info('Job {0}, task {1}: removing node backend {2} ' 'from group {3} node backends'.format( self.parent_job.id, self.id, node_backend, group)) group.remove_node_backend(node_backend) group.update_status_recursive() logger.info('Job {0}, task {1}: removed node backend {2} ' 'from group {3} node backends'.format( self.parent_job.id, self.id, node_backend, group))
def execute(self): try: hostname = cache.get_hostname_by_addr(self.host) except CacheUpstreamError: raise ValueError('Failed to resolve job host {}'.format(self.host)) nb_hostname_str = '{0}:{1}/{2}'.format( hostname, self.port, self.backend_id ).encode('utf-8') try: logger.info('Job {0}, task {1}: removing node backend {2} ' 'from group {3} history'.format( self.parent_job.id, self.id, nb_hostname_str, self.group)) infrastructure.detach_node( group_id=self.group, hostname=hostname, port=self.port, family=self.family, backend_id=self.backend_id, record_type=history.GroupStateRecord.HISTORY_RECORD_JOB, ) logger.info('Job {0}, task {1}: removed node backend {2} ' 'from group {3} history'.format( self.parent_job.id, self.id, nb_hostname_str, self.group)) except ValueError as e: # TODO: Think about changing ValueError to some dedicated exception # to differentiate between event when there is no such node in group # and an actual ValueError being raised logger.error('Job {0}, task {1}: failed to remove node backend {2} ' 'from group {3} history: {4}'.format( self.parent_job.id, self.id, nb_hostname_str, self.group, e)) pass group = self.group in storage.groups and storage.groups[self.group] or None nb_str = '{0}:{1}/{2}'.format(self.host, self.port, self.backend_id).encode('utf-8') node_backend = nb_str in storage.node_backends and storage.node_backends[nb_str] or None if group and node_backend and node_backend in group.node_backends: logger.info('Job {0}, task {1}: removing node backend {2} ' 'from group {3} node backends'.format( self.parent_job.id, self.id, node_backend, group)) group.remove_node_backend(node_backend) group.update_status_recursive() logger.info('Job {0}, task {1}: removed node backend {2} ' 'from group {3} node backends'.format( self.parent_job.id, self.id, node_backend, group))
def marker_format(self, marker): hostnames = [] for host in (self.src_host, self.dst_host): try: hostnames.append(cache.get_hostname_by_addr(host)) except CacheUpstreamError: raise RuntimeError('Failed to resolve host {0}'.format(host)) src_hostname, dst_hostname = hostnames return marker.format(group_id=str(self.group), src_host=self.src_host, src_hostname=src_hostname, src_backend_id=self.src_backend_id, src_port=str(self.src_port), src_base_path=self.src_base_path, dst_host=self.dst_host, dst_hostname=dst_hostname, dst_port=str(self.dst_port), dst_base_path=self.dst_base_path, dst_backend_id=self.dst_backend_id)
def marker_format(self, marker): hostnames = [] for host in (self.src_host, self.dst_host): try: hostnames.append(cache.get_hostname_by_addr(host)) except CacheUpstreamError: raise RuntimeError('Failed to resolve host {0}'.format(host)) src_hostname, dst_hostname = hostnames return marker.format( group_id=str(self.group), src_host=self.src_host, src_hostname=src_hostname, src_backend_id=self.src_backend_id, src_port=str(self.src_port), src_base_path=self.src_base_path, dst_host=self.dst_host, dst_hostname=dst_hostname, dst_port=str(self.dst_port), dst_base_path=self.dst_base_path, dst_backend_id=self.dst_backend_id)
def __node_in_group(self): group = self.group in storage.groups and storage.groups[ self.group] or None nb_str = '{0}:{1}/{2}'.format(self.host, self.port, self.backend_id).encode('utf-8') node_backend = nb_str in storage.node_backends and storage.node_backends[ nb_str] or None if group and node_backend: logger.debug('Job {0}, task {1}: checking node backend {2} ' 'with group {3} node backends: {4}'.format( self.parent_job.id, self.id, node_backend, self.group, group.node_backends)) nb_in_group = node_backend.group is group else: nb_in_group = False try: hostname = cache.get_hostname_by_addr(self.host) except CacheUpstreamError: raise ValueError('Failed to resolve job host {}'.format(self.host)) nb_in_history = infrastructure.node_backend_in_last_history_state( self.group, hostname, self.port, self.backend_id) logger.debug('Job {0}, task {1}: checking node backend {2} ' 'in group {3} history set: {4}'.format( self.parent_job.id, self.id, nb_str, self.group, nb_in_history)) if nb_in_group: logger.info('Job {0}, task {1}: node backend {2} is still ' 'in group {3}'.format(self.parent_job.id, self.id, nb_str, self.group)) if nb_in_history: logger.info('Job {0}, task {1}: node backend {2} is still ' 'in group\'s {3} history'.format( self.parent_job.id, self.id, nb_str, self.group)) return nb_in_group or nb_in_history
def create_tasks(self): group = storage.groups[self.group] src_group = storage.groups[self.src_group] self.check_node_backends(src_group) old_group_state = infrastructure.get_group_history(group.group_id)['nodes'][-1]['set'] if len(old_group_state) != 1: raise JobBrokenError('History of group {0} lists {1} ' 'node backends, 1 expected'.format(self.group, len(old_group_state))) old_host, old_port, old_family, old_backend_id, old_base_path = old_group_state[0][:5] if self.uncoupled_group: uncoupled_group = storage.groups[self.uncoupled_group] self.check_node_backends(uncoupled_group) dst_host = uncoupled_group.node_backends[0].node.host.addr dst_port = uncoupled_group.node_backends[0].node.port dst_family = uncoupled_group.node_backends[0].node.family dst_base_path = uncoupled_group.node_backends[0].base_path dst_backend_id = uncoupled_group.node_backends[0].backend_id else: # gettings dst_* from group history dst_host, dst_port, dst_backend_id = old_host, old_port, old_backend_id # TODO: Fix hardcoded family value dst_base_path, dst_family = old_base_path, old_family dst_node_backend = self.node_backend(dst_host, dst_port, dst_backend_id) group_file = (os.path.join(dst_base_path, self.GROUP_FILE_PATH) if self.GROUP_FILE_PATH else '') remove_path = '' if self.GROUP_FILE_DIR_MOVE_DST_RENAME and group_file: remove_path = os.path.join( dst_base_path, self.GROUP_FILE_DIR_MOVE_DST_RENAME) if self.uncoupled_group: for group_id in self.merged_groups: merged_group = storage.groups[group_id] merged_nb = merged_group.node_backends[0] merged_group_file = (os.path.join(merged_nb.base_path, self.GROUP_FILE_PATH) if self.GROUP_FILE_PATH else '') merged_path = '' if self.MERGE_GROUP_FILE_DIR_MOVE_SRC_RENAME and merged_group_file: merged_path = os.path.join( merged_nb.base_path, self.MERGE_GROUP_FILE_DIR_MOVE_SRC_RENAME) node_backend_str = self.node_backend(merged_nb.node.host.addr, merged_nb.node.port, merged_nb.backend_id) merged_group_file_marker = (os.path.join(merged_nb.base_path, self.MERGE_GROUP_FILE_MARKER_PATH) if self.MERGE_GROUP_FILE_MARKER_PATH else '') shutdown_cmd = infrastructure._disable_node_backend_cmd( merged_nb.node.host.addr, merged_nb.node.port, merged_nb.node.family, merged_nb.backend_id) params = {'node_backend': node_backend_str.encode('utf-8'), 'group': str(group_id), 'merged_to': str(self.uncoupled_group), 'remove_group_file': merged_group_file} if merged_group_file_marker: params['group_file_marker'] = merged_group_file_marker.format( dst_group_id=self.uncoupled_group, dst_backend_id=dst_backend_id) if merged_path: params['move_src'] = os.path.dirname(merged_group_file) params['move_dst'] = merged_path task = NodeStopTask.new(self, group=group_id, uncoupled=True, host=merged_nb.node.host.addr, cmd=shutdown_cmd, params=params) self.tasks.append(task) reconfigure_cmd = infrastructure._reconfigure_node_cmd( merged_nb.node.host.addr, merged_nb.node.port, merged_nb.node.family) task = MinionCmdTask.new(self, host=merged_nb.node.host.addr, cmd=reconfigure_cmd, params={'node_backend': node_backend_str.encode('utf-8')}) self.tasks.append(task) task = HistoryRemoveNodeTask.new(self, group=group_id, host=merged_nb.node.host.addr, port=merged_nb.node.port, backend_id=merged_nb.backend_id) self.tasks.append(task) # destination backend should be stopped shutdown_cmd = infrastructure._disable_node_backend_cmd( dst_host, dst_port, dst_family, dst_backend_id) params = {'node_backend': dst_node_backend.encode('utf-8'), 'group': str(self.uncoupled_group)} if remove_path: params['move_src'] = os.path.join(os.path.dirname(group_file)) params['move_dst'] = remove_path task = NodeStopTask.new(self, group=self.uncoupled_group, uncoupled=True, host=dst_host, cmd=shutdown_cmd, params=params) self.tasks.append(task) src_backend = src_group.node_backends[0] restore_backend = group.node_backends and group.node_backends[0] make_readonly = (src_backend is not restore_backend and src_backend.status == storage.Status.OK) mark_src_backend = self.make_path( self.BACKEND_DOWN_MARKER, base_path=src_backend.base_path).format( backend_id=src_backend.backend_id) if make_readonly: make_readonly_cmd = infrastructure._make_readonly_node_backend_cmd( src_backend.node.host.addr, src_backend.node.port, src_backend.node.family, src_backend.backend_id) task = MinionCmdTask.new(self, host=src_backend.node.host.addr, cmd=make_readonly_cmd, params={'node_backend': self.node_backend( src_backend.node.host.addr, src_backend.node.port, src_backend.backend_id).encode('utf-8'), 'mark_backend': mark_src_backend}) self.tasks.append(task) move_cmd = infrastructure.move_group_cmd( src_host=src_group.node_backends[0].node.host.addr, src_path=src_group.node_backends[0].base_path, src_family=src_group.node_backends[0].node.family, dst_path=dst_base_path) params = {'group': str(self.group), 'group_file': group_file} if remove_path and self.uncoupled_group: params['remove_path'] = remove_path task = RsyncBackendTask.new(self, host=dst_host, src_host=old_host, group=self.group, cmd=move_cmd, node_backend=self.node_backend(old_host, old_port, old_backend_id), params=params) self.tasks.append(task) additional_files = config.get('restore', {}).get('restore_additional_files', []) for src_file_tpl, dst_file_path in additional_files: rsync_cmd = infrastructure.move_group_cmd( src_host=src_group.node_backends[0].node.host.addr, src_path=src_group.node_backends[0].base_path, src_family=src_group.node_backends[0].node.family, dst_path=os.path.join(dst_base_path, dst_file_path), file_tpl=src_file_tpl) params = {'group': str(self.group)} task = MinionCmdTask.new(self, host=dst_host, group=self.group, cmd=rsync_cmd, params=params) self.tasks.append(task) if (restore_backend and restore_backend.status in storage.NodeBackend.ACTIVE_STATUSES): nb = group.node_backends[0] shutdown_cmd = infrastructure._disable_node_backend_cmd( nb.node.host.addr, nb.node.port, nb.node.family, nb.backend_id) group_file = (os.path.join(nb.base_path, self.GROUP_FILE_PATH) if self.GROUP_FILE_PATH else '') group_file_marker = (os.path.join(nb.base_path, self.GROUP_FILE_MARKER_PATH) if self.GROUP_FILE_MARKER_PATH else '') group_file_marker_fmt = group_file_marker.format( group_id=str(self.group), src_host=nb.node.host.addr, src_hostname=cache.get_hostname_by_addr( nb.node.host.addr), src_backend_id=nb.backend_id, src_port=str(nb.node.port), src_base_path=nb.base_path, dst_host=dst_host, dst_hostname=cache.get_hostname_by_addr(dst_host), dst_port=str(dst_port), dst_base_path=dst_base_path, dst_backend_id=dst_backend_id) params = {'node_backend': str(nb).encode('utf-8'), 'group': str(self.group), 'group_file_marker': group_file_marker_fmt, 'remove_group_file': group_file} if self.GROUP_FILE_DIR_MOVE_SRC_RENAME and group_file: params['move_src'] = os.path.join(os.path.dirname(group_file)) params['move_dst'] = os.path.join( nb.base_path, self.GROUP_FILE_DIR_MOVE_SRC_RENAME) task = NodeStopTask.new(self, group=self.group, host=nb.node.host.addr, cmd=shutdown_cmd, params=params) self.tasks.append(task) reconfigure_cmd = infrastructure._reconfigure_node_cmd( dst_host, dst_port, dst_family) task = MinionCmdTask.new(self, host=dst_host, cmd=reconfigure_cmd, params={'node_backend': dst_node_backend.encode('utf-8')}) self.tasks.append(task) if self.uncoupled_group: task = HistoryRemoveNodeTask.new(self, group=self.group, host=old_host, port=old_port, backend_id=old_backend_id) self.tasks.append(task) task = HistoryRemoveNodeTask.new(self, group=self.uncoupled_group, host=dst_host, port=dst_port, backend_id=dst_backend_id) self.tasks.append(task) start_cmd = infrastructure._enable_node_backend_cmd( dst_host, dst_port, dst_family, dst_backend_id) task = MinionCmdTask.new(self, host=dst_host, cmd=start_cmd, params={'node_backend': dst_node_backend.encode('utf-8')}) self.tasks.append(task) if make_readonly: make_writable_cmd = infrastructure._make_writable_node_backend_cmd( src_backend.node.host.addr, src_backend.node.port, src_backend.node.family, src_backend.backend_id) task = MinionCmdTask.new(self, host=src_backend.node.host.addr, cmd=make_writable_cmd, params={'node_backend': self.node_backend( src_backend.node.host.addr, src_backend.node.port, src_backend.backend_id).encode('utf-8'), 'unmark_backend': mark_src_backend, 'success_codes': [self.DNET_CLIENT_ALREADY_IN_PROGRESS]}) self.tasks.append(task) # This start command is executed for the case when elliptics # was restarted between make_readonly and make_writable commands start_cmd = infrastructure._enable_node_backend_cmd( src_backend.node.host.addr, src_backend.node.port, src_backend.node.family, src_backend.backend_id) task = MinionCmdTask.new(self, host=src_backend.node.host.addr, cmd=start_cmd, params={'node_backend': str(src_backend).encode('utf-8'), 'success_codes': [self.DNET_CLIENT_ALREADY_IN_PROGRESS]}) self.tasks.append(task)
def human_dump(self): data = super(HistoryRemoveNodeTask, self).human_dump() data['hostname'] = cache.get_hostname_by_addr(data['host'], strict=False) return data
def human_dump(self): data = super(RecoverDcJob, self).human_dump() data['hostname'] = cache.get_hostname_by_addr(data['host'], strict=False) return data
def create_tasks(self): group = storage.groups[self.group] src_group = storage.groups[self.src_group] self.check_node_backends(src_group) group_history = infrastructure.get_group_history(group.group_id) old_group_node_backends_set = group_history.nodes and group_history.nodes[-1].set or [] if old_group_node_backends_set: old_node_backend = old_group_node_backends_set[0] old_host = cache.get_ip_address_by_host(old_node_backend.hostname) old_port = old_node_backend.port old_family = old_node_backend.family old_backend_id = old_node_backend.backend_id old_base_path = old_node_backend.path if self.uncoupled_group: uncoupled_group = storage.groups[self.uncoupled_group] self.check_node_backends(uncoupled_group) dst_host = uncoupled_group.node_backends[0].node.host.addr dst_port = uncoupled_group.node_backends[0].node.port dst_family = uncoupled_group.node_backends[0].node.family dst_base_path = uncoupled_group.node_backends[0].base_path dst_backend_id = uncoupled_group.node_backends[0].backend_id else: if len(old_group_node_backends_set) != 1: raise JobBrokenError( 'History of group {0} lists {1} node backends, 1 expected'.format( self.group, len(old_group_node_backends_set) ) ) # gettings dst_* from group history dst_host, dst_port, dst_backend_id = old_host, old_port, old_backend_id # TODO: Fix hardcoded family value dst_base_path, dst_family = old_base_path, old_family dst_node_backend = self.node_backend(dst_host, dst_port, dst_backend_id) group_file = (os.path.join(dst_base_path, self.GROUP_FILE_PATH) if self.GROUP_FILE_PATH else '') remove_path = '' if self.GROUP_FILE_DIR_MOVE_DST_RENAME and group_file: remove_path = os.path.join( dst_base_path, self.GROUP_FILE_DIR_MOVE_DST_RENAME) if self.uncoupled_group: for group_id in self.merged_groups: merged_group = storage.groups[group_id] merged_nb = merged_group.node_backends[0] merged_group_file = (os.path.join(merged_nb.base_path, self.GROUP_FILE_PATH) if self.GROUP_FILE_PATH else '') merged_path = '' if self.MERGE_GROUP_FILE_DIR_MOVE_SRC_RENAME and merged_group_file: merged_path = os.path.join( merged_nb.base_path, self.MERGE_GROUP_FILE_DIR_MOVE_SRC_RENAME) node_backend_str = self.node_backend(merged_nb.node.host.addr, merged_nb.node.port, merged_nb.backend_id) merged_group_file_marker = (os.path.join(merged_nb.base_path, self.MERGE_GROUP_FILE_MARKER_PATH) if self.MERGE_GROUP_FILE_MARKER_PATH else '') shutdown_cmd = infrastructure._disable_node_backend_cmd( merged_nb.node.host.addr, merged_nb.node.port, merged_nb.node.family, merged_nb.backend_id) params = {'node_backend': node_backend_str.encode('utf-8'), 'group': str(group_id), 'merged_to': str(self.uncoupled_group), 'remove_group_file': merged_group_file} if merged_group_file_marker: params['group_file_marker'] = merged_group_file_marker.format( dst_group_id=self.uncoupled_group, dst_backend_id=dst_backend_id) if merged_path: params['move_src'] = os.path.dirname(merged_group_file) params['move_dst'] = merged_path task = NodeStopTask.new(self, group=group_id, uncoupled=True, host=merged_nb.node.host.addr, cmd=shutdown_cmd, params=params) self.tasks.append(task) reconfigure_cmd = infrastructure._reconfigure_node_cmd( merged_nb.node.host.addr, merged_nb.node.port, merged_nb.node.family) task = MinionCmdTask.new(self, host=merged_nb.node.host.addr, cmd=reconfigure_cmd, params={'node_backend': node_backend_str.encode('utf-8')}) self.tasks.append(task) task = HistoryRemoveNodeTask.new( self, group=group_id, host=merged_nb.node.host.addr, port=merged_nb.node.port, family=merged_nb.node.family, backend_id=merged_nb.backend_id, ) self.tasks.append(task) # destination backend should be stopped shutdown_cmd = infrastructure._disable_node_backend_cmd( dst_host, dst_port, dst_family, dst_backend_id) params = {'node_backend': dst_node_backend.encode('utf-8'), 'group': str(self.uncoupled_group)} if remove_path: params['move_src'] = os.path.join(os.path.dirname(group_file)) params['move_dst'] = remove_path task = NodeStopTask.new(self, group=self.uncoupled_group, uncoupled=True, host=dst_host, cmd=shutdown_cmd, params=params) self.tasks.append(task) src_backend = src_group.node_backends[0] restore_backend = group.node_backends and group.node_backends[0] make_readonly = (src_backend is not restore_backend and src_backend.status == storage.Status.OK) mark_src_backend = self.make_path( self.BACKEND_DOWN_MARKER, base_path=src_backend.base_path).format( backend_id=src_backend.backend_id) if make_readonly: make_readonly_cmd = infrastructure._make_readonly_node_backend_cmd( src_backend.node.host.addr, src_backend.node.port, src_backend.node.family, src_backend.backend_id) task = MinionCmdTask.new(self, host=src_backend.node.host.addr, cmd=make_readonly_cmd, params={'node_backend': self.node_backend( src_backend.node.host.addr, src_backend.node.port, src_backend.backend_id).encode('utf-8'), 'mark_backend': mark_src_backend}) self.tasks.append(task) move_cmd = infrastructure.move_group_cmd( src_host=src_group.node_backends[0].node.host.addr, src_path=src_group.node_backends[0].base_path, src_family=src_group.node_backends[0].node.family, dst_path=dst_base_path) ids_file = (os.path.join(dst_base_path, self.IDS_FILE_PATH) if self.IDS_FILE_PATH else '') params = {'group': str(self.group), 'group_file': group_file, 'ids': ids_file} if remove_path and self.uncoupled_group: params['remove_path'] = remove_path check_node_backend = None if old_group_node_backends_set: check_node_backend = self.node_backend(old_host, old_port, old_backend_id) task = RsyncBackendTask.new( self, host=dst_host, src_host=src_group.node_backends[0].node.host.addr, group=self.group, cmd=move_cmd, node_backend=check_node_backend, params=params ) self.tasks.append(task) additional_files = config.get('restore', {}).get('restore_additional_files', []) for src_file_tpl, dst_file_path in additional_files: rsync_cmd = infrastructure.move_group_cmd( src_host=src_group.node_backends[0].node.host.addr, src_path=src_group.node_backends[0].base_path, src_family=src_group.node_backends[0].node.family, dst_path=os.path.join(dst_base_path, dst_file_path), file_tpl=src_file_tpl) params = {'group': str(self.group)} task = MinionCmdTask.new(self, host=dst_host, group=self.group, cmd=rsync_cmd, params=params) self.tasks.append(task) if (restore_backend and restore_backend.status in storage.NodeBackend.ACTIVE_STATUSES): nb = group.node_backends[0] shutdown_cmd = infrastructure._disable_node_backend_cmd( nb.node.host.addr, nb.node.port, nb.node.family, nb.backend_id) group_file = (os.path.join(nb.base_path, self.GROUP_FILE_PATH) if self.GROUP_FILE_PATH else '') group_file_marker = (os.path.join(nb.base_path, self.GROUP_FILE_MARKER_PATH) if self.GROUP_FILE_MARKER_PATH else '') hostnames = [] for host in (nb.node.host.addr, dst_host): try: hostnames.append(cache.get_hostname_by_addr(host)) except CacheUpstreamError: raise RuntimeError('Failed to resolve host {0}'.format(host)) src_hostname, dst_hostname = hostnames group_file_marker_fmt = group_file_marker.format( group_id=str(self.group), src_host=nb.node.host.addr, src_hostname=src_hostname, src_backend_id=nb.backend_id, src_port=str(nb.node.port), src_base_path=nb.base_path, dst_host=dst_host, dst_hostname=dst_hostname, dst_port=str(dst_port), dst_base_path=dst_base_path, dst_backend_id=dst_backend_id) params = {'node_backend': str(nb).encode('utf-8'), 'group': str(self.group), 'group_file_marker': group_file_marker_fmt, 'remove_group_file': group_file} if self.GROUP_FILE_DIR_MOVE_SRC_RENAME and group_file: params['move_src'] = os.path.join(os.path.dirname(group_file)) params['move_dst'] = os.path.join( nb.base_path, self.GROUP_FILE_DIR_MOVE_SRC_RENAME) task = NodeStopTask.new(self, group=self.group, host=nb.node.host.addr, cmd=shutdown_cmd, params=params) self.tasks.append(task) reconfigure_cmd = infrastructure._reconfigure_node_cmd( dst_host, dst_port, dst_family) task = MinionCmdTask.new(self, host=dst_host, cmd=reconfigure_cmd, params={'node_backend': dst_node_backend.encode('utf-8')}) self.tasks.append(task) if self.uncoupled_group: if old_group_node_backends_set: task = HistoryRemoveNodeTask.new( self, group=self.group, host=old_host, port=old_port, family=old_family, backend_id=old_backend_id, ) self.tasks.append(task) task = HistoryRemoveNodeTask.new( self, group=self.uncoupled_group, host=dst_host, port=dst_port, family=dst_family, backend_id=dst_backend_id, ) self.tasks.append(task) start_cmd = infrastructure._enable_node_backend_cmd( dst_host, dst_port, dst_family, dst_backend_id) task = MinionCmdTask.new(self, host=dst_host, cmd=start_cmd, params={'node_backend': dst_node_backend.encode('utf-8')}) self.tasks.append(task) if make_readonly: make_writable_cmd = infrastructure._make_writable_node_backend_cmd( src_backend.node.host.addr, src_backend.node.port, src_backend.node.family, src_backend.backend_id) task = MinionCmdTask.new(self, host=src_backend.node.host.addr, cmd=make_writable_cmd, params={'node_backend': self.node_backend( src_backend.node.host.addr, src_backend.node.port, src_backend.backend_id).encode('utf-8'), 'unmark_backend': mark_src_backend, 'success_codes': [self.DNET_CLIENT_ALREADY_IN_PROGRESS]}) self.tasks.append(task) # This start command is executed for the case when elliptics # was restarted between make_readonly and make_writable commands start_cmd = infrastructure._enable_node_backend_cmd( src_backend.node.host.addr, src_backend.node.port, src_backend.node.family, src_backend.backend_id) task = MinionCmdTask.new(self, host=src_backend.node.host.addr, cmd=start_cmd, params={'node_backend': str(src_backend).encode('utf-8'), 'success_codes': [self.DNET_CLIENT_ALREADY_IN_PROGRESS]}) self.tasks.append(task)
def parents(self): return cache.get_host_tree( cache.get_hostname_by_addr(self.addr))
def human_dump(self): data = super(MinionCmdTask, self).human_dump() data['hostname'] = cache.get_hostname_by_addr(data['host'], strict=False) return data
def human_dump(self): data = super(RecoverDcJob, self).human_dump() data['hostname'] = cache.get_hostname_by_addr(data['host']) return data
def hostname(self): return cache.get_hostname_by_addr(self.addr)