def find_and_process(self): src_filename = time.strftime(self.filename_format) working_dir = os.path.join(self.target_dir, '.stats_tmp') shutil.rmtree(working_dir, ignore_errors=True) mkdirs(working_dir) tmp_filename = os.path.join(working_dir, src_filename) hasher = hashlib.md5() with open(tmp_filename, 'wb') as statfile: # csv has the following columns: # Account Name, Container Count, Object Count, Bytes Used for device in os.listdir(self.devices): if self.mount_check and not check_mount(self.devices, device): self.logger.error( _("Device %s is not mounted, skipping.") % device) continue accounts = os.path.join(self.devices, device, account_server_data_dir) if not os.path.exists(accounts): self.logger.debug(_("Path %s does not exist, skipping.") % accounts) continue for root, dirs, files in os.walk(accounts, topdown=False): for filename in files: if filename.endswith('.db'): db_path = os.path.join(root, filename) broker = AccountBroker(db_path) if not broker.is_deleted(): (account_name, _junk, _junk, _junk, container_count, object_count, bytes_used, _junk, _junk) = broker.get_info() line_data = '"%s",%d,%d,%d\n' % ( account_name, container_count, object_count, bytes_used) statfile.write(line_data) hasher.update(line_data) file_hash = hasher.hexdigest() hash_index = src_filename.find('*') if hash_index < 0: # if there is no * in the target filename, the uploader probably # won't work because we are crafting a filename that doesn't # fit the pattern src_filename = '_'.join([src_filename, file_hash]) else: parts = src_filename[:hash_index], src_filename[hash_index + 1:] src_filename = ''.join([parts[0], file_hash, parts[1]]) renamer(tmp_filename, os.path.join(self.target_dir, src_filename)) shutil.rmtree(working_dir, ignore_errors=True)
def get_data(self, db_path): """ Data for generated csv has the following columns: Account Hash, Container Count, Object Count, Bytes Used :raises sqlite3.Error: does not catch errors connecting to db """ line_data = None broker = AccountBroker(db_path) if not broker.is_deleted(): info = broker.get_info() line_data = '"%s",%d,%d,%d\n' % (info['account'], info['container_count'], info['object_count'], info['bytes_used']) return line_data
def find_and_process(self): src_filename = time.strftime(self.filename_format) working_dir = os.path.join(self.target_dir, '.stats_tmp') shutil.rmtree(working_dir, ignore_errors=True) mkdirs(working_dir) tmp_filename = os.path.join(working_dir, src_filename) hasher = hashlib.md5() with open(tmp_filename, 'wb') as statfile: # csv has the following columns: # Account Name, Container Count, Object Count, Bytes Used for device in os.listdir(self.devices): if self.mount_check and not check_mount(self.devices, device): self.logger.error( _("Device %s is not mounted, skipping.") % device) continue accounts = os.path.join(self.devices, device, account_server_data_dir) if not os.path.exists(accounts): self.logger.debug( _("Path %s does not exist, skipping.") % accounts) continue for root, dirs, files in os.walk(accounts, topdown=False): for filename in files: if filename.endswith('.db'): db_path = os.path.join(root, filename) broker = AccountBroker(db_path) if not broker.is_deleted(): (account_name, _junk, _junk, _junk, container_count, object_count, bytes_used, _junk, _junk) = broker.get_info() line_data = '"%s",%d,%d,%d\n' % ( account_name, container_count, object_count, bytes_used) statfile.write(line_data) hasher.update(line_data) file_hash = hasher.hexdigest() hash_index = src_filename.find('*') if hash_index < 0: # if there is no * in the target filename, the uploader probably # won't work because we are crafting a filename that doesn't # fit the pattern src_filename = '_'.join([src_filename, file_hash]) else: parts = src_filename[:hash_index], src_filename[hash_index + 1:] src_filename = ''.join([parts[0], file_hash, parts[1]]) renamer(tmp_filename, os.path.join(self.target_dir, src_filename)) shutil.rmtree(working_dir, ignore_errors=True)
def account_audit(self, path): """ Audits the given account path :param path: the path to an account db """ try: if not path.endswith(".db"): return broker = AccountBroker(path) if not broker.is_deleted(): info = broker.get_info() self.account_passes += 1 self.logger.debug(_("Audit passed for %s") % broker.db_file) except Exception: self.account_failures += 1 self.logger.exception(_("ERROR Could not get account info %s"), (broker.db_file))
def account_audit(self, path): """ Audits the given account path :param path: the path to an account db """ try: if not path.endswith('.db'): return broker = AccountBroker(path) if not broker.is_deleted(): info = broker.get_info() self.account_passes += 1 self.logger.debug(_('Audit passed for %s') % broker.db_file) except Exception: self.account_failures += 1 self.logger.exception(_('ERROR Could not get account info %s'), (broker.db_file))
def account_audit(self, path): """ Audits the given account path :param path: the path to an account db """ start_time = time.time() try: broker = AccountBroker(path) if not broker.is_deleted(): broker.get_info() self.logger.increment('passes') self.account_passes += 1 self.logger.debug(_('Audit passed for %s') % broker.db_file) except (Exception, Timeout): self.logger.increment('failures') self.account_failures += 1 self.logger.exception(_('ERROR Could not get account info %s'), (broker.db_file)) self.logger.timing_since('timing', start_time)
def account_audit(self, path): """ Audits the given account path :param path: the path to an account db """ start_time = time.time() try: broker = AccountBroker(path) if not broker.is_deleted(): broker.get_info() self.logger.increment('passes') self.account_passes += 1 self.logger.debug(_('Audit passed for %s') % broker.db_file) except (Exception, Timeout): self.logger.increment('failures') self.account_failures += 1 self.logger.exception(_('ERROR Could not get account info %s'), (broker.db_file)) self.logger.timing_since('timing', start_time)
def account_audit(self, path): """ Audits the given account path :param path: the path to an account db """ start_time = time.time() try: if not path.endswith(".db"): return broker = AccountBroker(path) if not broker.is_deleted(): info = broker.get_info() self.logger.increment("passes") self.account_passes += 1 self.logger.debug(_("Audit passed for %s") % broker.db_file) except (Exception, Timeout): self.logger.increment("failures") self.account_failures += 1 self.logger.exception(_("ERROR Could not get account info %s"), (broker.db_file)) self.logger.timing_since("timing", start_time)