def test_account_stat_get_data(self): stat = db_stats_collector.AccountStatsCollector(self.conf) account_db = AccountBroker("%s/acc.db" % self.accounts, account='test_acc') account_db.initialize() account_db.put_container('test_container', time.time(), None, 10, 1000) info = stat.get_data("%s/acc.db" % self.accounts) self.assertEquals('''"test_acc",1,10,1000\n''', info)
def find_and_process(self): src_filename = time.strftime(self.filename_format) working_dir = os.path.join(self.target_dir, '.stats_tmp') shutil.rmtree(working_dir, ignore_errors=True) mkdirs(working_dir) tmp_filename = os.path.join(working_dir, src_filename) hasher = hashlib.md5() with open(tmp_filename, 'wb') as statfile: # csv has the following columns: # Account Name, Container Count, Object Count, Bytes Used for device in os.listdir(self.devices): if self.mount_check and not check_mount(self.devices, device): self.logger.error( _("Device %s is not mounted, skipping.") % device) continue accounts = os.path.join(self.devices, device, account_server_data_dir) if not os.path.exists(accounts): self.logger.debug(_("Path %s does not exist, skipping.") % accounts) continue for root, dirs, files in os.walk(accounts, topdown=False): for filename in files: if filename.endswith('.db'): db_path = os.path.join(root, filename) broker = AccountBroker(db_path) if not broker.is_deleted(): (account_name, _junk, _junk, _junk, container_count, object_count, bytes_used, _junk, _junk) = broker.get_info() line_data = '"%s",%d,%d,%d\n' % ( account_name, container_count, object_count, bytes_used) statfile.write(line_data) hasher.update(line_data) file_hash = hasher.hexdigest() hash_index = src_filename.find('*') if hash_index < 0: # if there is no * in the target filename, the uploader probably # won't work because we are crafting a filename that doesn't # fit the pattern src_filename = '_'.join([src_filename, file_hash]) else: parts = src_filename[:hash_index], src_filename[hash_index + 1:] src_filename = ''.join([parts[0], file_hash, parts[1]]) renamer(tmp_filename, os.path.join(self.target_dir, src_filename)) shutil.rmtree(working_dir, ignore_errors=True)
def _get_account_broker(self, drive, part, account, **kwargs): hsh = hash_path(account) db_dir = storage_directory(DATADIR, part, hsh) db_path = os.path.join(self.root, drive, db_dir, hsh + '.db') kwargs.setdefault('account', account) kwargs.setdefault('logger', self.logger) return AccountBroker(db_path, **kwargs)
def get_data(self, db_path): """ Data for generated csv has the following columns: Account Hash, Container Count, Object Count, Bytes Used :raises sqlite3.Error: does not catch errors connecting to db """ line_data = None broker = AccountBroker(db_path) if not broker.is_deleted(): info = broker.get_info() line_data = '"%s",%d,%d,%d\n' % (info['account'], info['container_count'], info['object_count'], info['bytes_used']) return line_data
def find_and_process(self): src_filename = time.strftime(self.filename_format) working_dir = os.path.join(self.target_dir, '.stats_tmp') shutil.rmtree(working_dir, ignore_errors=True) mkdirs(working_dir) tmp_filename = os.path.join(working_dir, src_filename) hasher = hashlib.md5() with open(tmp_filename, 'wb') as statfile: # csv has the following columns: # Account Name, Container Count, Object Count, Bytes Used for device in os.listdir(self.devices): if self.mount_check and not check_mount(self.devices, device): self.logger.error( _("Device %s is not mounted, skipping.") % device) continue accounts = os.path.join(self.devices, device, account_server_data_dir) if not os.path.exists(accounts): self.logger.debug( _("Path %s does not exist, skipping.") % accounts) continue for root, dirs, files in os.walk(accounts, topdown=False): for filename in files: if filename.endswith('.db'): db_path = os.path.join(root, filename) broker = AccountBroker(db_path) if not broker.is_deleted(): (account_name, _junk, _junk, _junk, container_count, object_count, bytes_used, _junk, _junk) = broker.get_info() line_data = '"%s",%d,%d,%d\n' % ( account_name, container_count, object_count, bytes_used) statfile.write(line_data) hasher.update(line_data) file_hash = hasher.hexdigest() hash_index = src_filename.find('*') if hash_index < 0: # if there is no * in the target filename, the uploader probably # won't work because we are crafting a filename that doesn't # fit the pattern src_filename = '_'.join([src_filename, file_hash]) else: parts = src_filename[:hash_index], src_filename[hash_index + 1:] src_filename = ''.join([parts[0], file_hash, parts[1]]) renamer(tmp_filename, os.path.join(self.target_dir, src_filename)) shutil.rmtree(working_dir, ignore_errors=True)
def account_audit(self, path): """ Audits the given account path :param path: the path to an account db """ try: if not path.endswith(".db"): return broker = AccountBroker(path) if not broker.is_deleted(): info = broker.get_info() self.account_passes += 1 self.logger.debug(_("Audit passed for %s") % broker.db_file) except Exception: self.account_failures += 1 self.logger.exception(_("ERROR Could not get account info %s"), (broker.db_file))
def _get_account_broker(self, drive, part, account): if self.fs_object: return DiskAccount(self.root, account, self.fs_object) hsh = hash_path(account) db_dir = storage_directory(DATADIR, part, hsh) db_path = os.path.join(self.root, drive, db_dir, hsh + '.db') return AccountBroker(db_path, account=account, logger=self.logger)
def account_audit(self, path): """ Audits the given account path :param path: the path to an account db """ try: if not path.endswith('.db'): return broker = AccountBroker(path) if not broker.is_deleted(): info = broker.get_info() self.account_passes += 1 self.logger.debug(_('Audit passed for %s') % broker.db_file) except Exception: self.account_failures += 1 self.logger.exception(_('ERROR Could not get account info %s'), (broker.db_file))
def reap_device(self, device): """ Called once per pass for each device on the server. This will scan the accounts directory for the device, looking for partitions this device is the primary for, then looking for account databases that are marked status=DELETED and still have containers and calling :func:`reap_account`. Account databases marked status=DELETED that no longer have containers will eventually be permanently removed by the reclaim process within the account replicator (see :mod:`swift.db_replicator`). :param device: The device to look for accounts to be deleted. """ datadir = os.path.join(self.devices, device, DATADIR) if not os.path.exists(datadir): return for partition in os.listdir(datadir): partition_path = os.path.join(datadir, partition) if not partition.isdigit(): continue nodes = self.get_account_ring().get_part_nodes(int(partition)) if nodes[0]['ip'] not in self.myips or \ not os.path.isdir(partition_path): continue for suffix in os.listdir(partition_path): suffix_path = os.path.join(partition_path, suffix) if not os.path.isdir(suffix_path): continue for hsh in os.listdir(suffix_path): hsh_path = os.path.join(suffix_path, hsh) if not os.path.isdir(hsh_path): continue for fname in sorted(os.listdir(hsh_path), reverse=True): if fname.endswith('.ts'): break elif fname.endswith('.db'): self.start_time = time() broker = \ AccountBroker(os.path.join(hsh_path, fname)) if broker.is_status_deleted() and \ not broker.empty(): self.reap_account(broker, partition, nodes)
def account_audit(self, path): """ Audits the given account path :param path: the path to an account db """ start_time = time.time() try: broker = AccountBroker(path) if not broker.is_deleted(): broker.get_info() self.logger.increment('passes') self.account_passes += 1 self.logger.debug(_('Audit passed for %s') % broker.db_file) except (Exception, Timeout): self.logger.increment('failures') self.account_failures += 1 self.logger.exception(_('ERROR Could not get account info %s'), (broker.db_file)) self.logger.timing_since('timing', start_time)
def account_audit(self, path): """ Audits the given account path :param path: the path to an account db """ start_time = time.time() try: if not path.endswith(".db"): return broker = AccountBroker(path) if not broker.is_deleted(): info = broker.get_info() self.logger.increment("passes") self.account_passes += 1 self.logger.debug(_("Audit passed for %s") % broker.db_file) except (Exception, Timeout): self.logger.increment("failures") self.account_failures += 1 self.logger.exception(_("ERROR Could not get account info %s"), (broker.db_file)) self.logger.timing_since("timing", start_time)
def _move_file(self, filename, filetype): if filetype == 'accounts': broker = AccountBroker(filename) info = broker.get_info() elif filetype == 'containers': broker = ContainerBroker(filename) info = broker.get_info() elif filetype == 'objects': info = self._get_acc_cont_obj(filename) else: raise Exception acc = info.get('account') cont = info.get('container') obj = info.get('object') partition, _nodes = self.ring.get_nodes(acc, cont, obj) # replace the old partition value with the new one # old name like '/a/b/objects/123/c/d' # new name like '/a/b/objects/456/c/d' filename_parts = filename.split('/') part_pos = filename_parts.index(filetype) filename_parts[part_pos+1] = str(partition) newname = '/'.join(filename_parts) dst_dir = os.path.dirname(newname) try: os.makedirs(dst_dir) logging.info("mkdir %s" % dst_dir) except OSError as ex: logging.info("mkdir %s failed: %s" % (dst_dir, ex)) try: os.rename(filename, newname) logging.info("moved %s -> %s" % (filename, newname)) except OSError as ex: logging.warning("FAILED TO MOVE %s -> %s" % (filename, newname))
def _gen_account_stat(self): stat = db_stats_collector.AccountStatsCollector(self.conf) output_data = set() for i in range(10): account_db = AccountBroker("%s/stats-201001010%s-%s.db" % (self.accounts, i, uuid.uuid4().hex), account='test_acc_%s' % i) account_db.initialize() account_db.put_container('test_container', time.time(), None, 10, 1000) # this will "commit" the data account_db.get_info() output_data.add('''"test_acc_%s",1,10,1000''' % i), self.assertEqual(len(output_data), 10) return stat, output_data