def audit_location_generator(devices, datadir, suffix='', mount_check=True, logger=None): ''' Given a devices path and a data directory, yield (path, device, partition) for all files in that directory :param devices: parent directory of the devices to be audited :param datadir: a directory located under self.devices. This should be one of the DATADIR constants defined in the account, container, and object servers. :param suffix: path name suffix required for all names returned :param mount_check: Flag to check if a mount check should be performed on devices :param logger: a logger object ''' device_dir = listdir(devices) # randomize devices in case of process restart before sweep completed shuffle(device_dir) for device in device_dir: if mount_check and not \ os.path.ismount(os.path.join(devices, device)): if logger: logger.debug(_('Skipping %s as it is not mounted'), device) continue datadir_path = os.path.join(devices, device, datadir) partitions = listdir(datadir_path) for partition in partitions: part_path = os.path.join(datadir_path, partition) try: suffixes = listdir(part_path) except OSError as e: if e.errno != errno.ENOTDIR: raise continue for asuffix in suffixes: suff_path = os.path.join(part_path, asuffix) try: hashes = listdir(suff_path) except OSError as e: if e.errno != errno.ENOTDIR: raise continue for hsh in hashes: hash_path = os.path.join(suff_path, hsh) try: files = sorted(listdir(hash_path), reverse=True) except OSError as e: if e.errno != errno.ENOTDIR: raise continue for fname in files: if suffix and not fname.endswith(suffix): continue path = os.path.join(hash_path, fname) yield path, device, partition
def audit_location_generator(devices, datadir, suffix='', mount_check=True, logger=None): ''' Given a devices path and a data directory, yield (path, device, partition) for all files in that directory :param devices: parent directory of the devices to be audited :param datadir: a directory located under self.devices. This should be one of the DATADIR constants defined in the account, container, and object servers. :param suffix: path name suffix required for all names returned :param mount_check: Flag to check if a mount check should be performed on devices :param logger: a logger object ''' device_dir = listdir(devices) # randomize devices in case of process restart before sweep completed shuffle(device_dir) for device in device_dir: if mount_check and not \ os.path.ismount(os.path.join(devices, device)): if logger: logger.debug( _('Skipping %s as it is not mounted'), device) continue datadir_path = os.path.join(devices, device, datadir) partitions = listdir(datadir_path) for partition in partitions: part_path = os.path.join(datadir_path, partition) try: suffixes = listdir(part_path) except OSError as e: if e.errno != errno.ENOTDIR: raise continue for asuffix in suffixes: suff_path = os.path.join(part_path, asuffix) try: hashes = listdir(suff_path) except OSError as e: if e.errno != errno.ENOTDIR: raise continue for hsh in hashes: hash_path = os.path.join(suff_path, hsh) try: files = sorted(listdir(hash_path), reverse=True) except OSError as e: if e.errno != errno.ENOTDIR: raise continue for fname in files: if suffix and not fname.endswith(suffix): continue path = os.path.join(hash_path, fname) yield path, device, partition
def audit_loop(self, parent, zbo_fps, override_devices=None, **kwargs): """Parallel audit loop""" self.clear_recon_cache('ALL') self.clear_recon_cache('ZBF') once = kwargs.get('mode') == 'once' kwargs['device_dirs'] = override_devices if parent: kwargs['zero_byte_fps'] = zbo_fps self.run_audit(**kwargs) else: pids = set() if self.conf_zero_byte_fps: zbf_pid = self.fork_child(zero_byte_fps=True, **kwargs) pids.add(zbf_pid) if self.concurrency == 1: # Audit all devices in 1 process pids.add(self.fork_child(**kwargs)) else: # Divide devices amongst parallel processes set by # self.concurrency. Total number of parallel processes # is self.concurrency + 1 if zero_byte_fps. parallel_proc = self.concurrency + 1 if \ self.conf_zero_byte_fps else self.concurrency device_list = list(override_devices) if override_devices else \ listdir(self.devices) shuffle(device_list) while device_list: pid = None if len(pids) == parallel_proc: pid = os.wait()[0] pids.discard(pid) if self.conf_zero_byte_fps and pid == zbf_pid and once: # If we're only running one pass and the ZBF scanner # finished, don't bother restarting it. zbf_pid = -100 elif self.conf_zero_byte_fps and pid == zbf_pid: # When we're running forever, the ZBF scanner must # be restarted as soon as it finishes. kwargs['device_dirs'] = override_devices # sleep between ZBF scanner forks self._sleep() zbf_pid = self.fork_child(zero_byte_fps=True, **kwargs) pids.add(zbf_pid) else: kwargs['device_dirs'] = [device_list.pop()] pids.add(self.fork_child(**kwargs)) while pids: pid = os.wait()[0] # ZBF scanner must be restarted as soon as it finishes # unless we're in run-once mode if self.conf_zero_byte_fps and pid == zbf_pid and \ len(pids) > 1 and not once: kwargs['device_dirs'] = override_devices # sleep between ZBF scanner forks zbf_pid = self.fork_child(zero_byte_fps=True, sleep_between_zbf_scanner=True, **kwargs) pids.add(zbf_pid) pids.discard(pid)
def audit_loop(self, parent, zbo_fps, override_devices=None, **kwargs): """Parallel audit loop""" self.clear_recon_cache('ALL') self.clear_recon_cache('ZBF') kwargs['device_dirs'] = override_devices if parent: kwargs['zero_byte_fps'] = zbo_fps self.run_audit(**kwargs) else: pids = [] if self.conf_zero_byte_fps: zbf_pid = self.fork_child(zero_byte_fps=True, **kwargs) pids.append(zbf_pid) if self.concurrency == 1: # Audit all devices in 1 process pids.append(self.fork_child(**kwargs)) else: # Divide devices amongst parallel processes set by # self.concurrency. Total number of parallel processes # is self.concurrency + 1 if zero_byte_fps. parallel_proc = self.concurrency + 1 if \ self.conf_zero_byte_fps else self.concurrency device_list = list(override_devices) if override_devices else \ listdir(self.devices) shuffle(device_list) while device_list: pid = None if len(pids) == parallel_proc: pid = os.wait()[0] pids.remove(pid) # ZBF scanner must be restarted as soon as it finishes if self.conf_zero_byte_fps and pid == zbf_pid: kwargs['device_dirs'] = override_devices # sleep between ZBF scanner forks self._sleep() zbf_pid = self.fork_child(zero_byte_fps=True, **kwargs) pids.append(zbf_pid) else: kwargs['device_dirs'] = [device_list.pop()] pids.append(self.fork_child(**kwargs)) while pids: pid = os.wait()[0] # ZBF scanner must be restarted as soon as it finishes if self.conf_zero_byte_fps and pid == zbf_pid and \ len(pids) > 1: kwargs['device_dirs'] = override_devices # sleep between ZBF scanner forks self._sleep() zbf_pid = self.fork_child(zero_byte_fps=True, **kwargs) pids.append(zbf_pid) pids.remove(pid)
def run_once(self, *args, **kwargs): """Run the object audit once""" # zero byte only command line option zbo_fps = kwargs.get("zero_byte_fps", 0) override_devices = list_from_csv(kwargs.get("devices")) # Remove bogus entries and duplicates from override_devices override_devices = list(set(listdir(self.devices)).intersection(set(override_devices))) parent = False if zbo_fps: # only start parent parent = True kwargs = {"mode": "once"} try: self.audit_loop(parent, zbo_fps, override_devices=override_devices, **kwargs) except (Exception, Timeout) as err: self.logger.exception(_("ERROR auditing: %s" % err))
def run_once(self, *args, **kwargs): """Run the object audit once""" # zero byte only command line option zbo_fps = kwargs.get('zero_byte_fps', 0) override_devices = list_from_csv(kwargs.get('devices')) # Remove bogus entries and duplicates from override_devices override_devices = list( set(listdir(self.devices)).intersection(set(override_devices))) parent = False if zbo_fps: # only start parent parent = True kwargs = {'mode': 'once'} try: self.audit_loop(parent, zbo_fps, override_devices=override_devices, **kwargs) except (Exception, Timeout) as err: self.logger.exception(_('ERROR auditing: %s'), err)