def _get_account_broker(self, drive, part, account, **kwargs): hsh = hash_path(account) db_dir = storage_directory(DATADIR, part, hsh) db_path = os.path.join(self.root, drive, db_dir, hsh + '.db') kwargs.setdefault('account', account) kwargs.setdefault('logger', self.logger) return AccountBroker(db_path, **kwargs)
def _get_account_broker(self, drive, part, account): if self.fs_object: return DiskAccount(self.root, account, self.fs_object) hsh = hash_path(account) db_dir = storage_directory(DATADIR, part, hsh) db_path = os.path.join(self.root, drive, db_dir, hsh + '.db') return AccountBroker(db_path, account=account, logger=self.logger)
def test_account_stat_get_data(self): stat = db_stats_collector.AccountStatsCollector(self.conf) account_db = AccountBroker("%s/acc.db" % self.accounts, account='test_acc') account_db.initialize() account_db.put_container('test_container', time.time(), None, 10, 1000) info = stat.get_data("%s/acc.db" % self.accounts) self.assertEquals('''"test_acc",1,10,1000\n''', info)
def get_data(self, db_path): """ Data for generated csv has the following columns: Account Hash, Container Count, Object Count, Bytes Used :raises sqlite3.Error: does not catch errors connecting to db """ line_data = None broker = AccountBroker(db_path) if not broker.is_deleted(): info = broker.get_info() line_data = '"%s",%d,%d,%d\n' % (info['account'], info['container_count'], info['object_count'], info['bytes_used']) return line_data
def _gen_account_stat(self): stat = db_stats_collector.AccountStatsCollector(self.conf) output_data = set() for i in range(10): account_db = AccountBroker("%s/stats-201001010%s-%s.db" % (self.accounts, i, uuid.uuid4().hex), account='test_acc_%s' % i) account_db.initialize() account_db.put_container('test_container', time.time(), None, 10, 1000) # this will "commit" the data account_db.get_info() output_data.add('''"test_acc_%s",1,10,1000''' % i), self.assertEqual(len(output_data), 10) return stat, output_data
def find_and_process(self): src_filename = time.strftime(self.filename_format) working_dir = os.path.join(self.target_dir, '.stats_tmp') shutil.rmtree(working_dir, ignore_errors=True) mkdirs(working_dir) tmp_filename = os.path.join(working_dir, src_filename) hasher = hashlib.md5() with open(tmp_filename, 'wb') as statfile: # csv has the following columns: # Account Name, Container Count, Object Count, Bytes Used for device in os.listdir(self.devices): if self.mount_check and not check_mount(self.devices, device): self.logger.error( _("Device %s is not mounted, skipping.") % device) continue accounts = os.path.join(self.devices, device, account_server_data_dir) if not os.path.exists(accounts): self.logger.debug( _("Path %s does not exist, skipping.") % accounts) continue for root, dirs, files in os.walk(accounts, topdown=False): for filename in files: if filename.endswith('.db'): db_path = os.path.join(root, filename) broker = AccountBroker(db_path) if not broker.is_deleted(): (account_name, _junk, _junk, _junk, container_count, object_count, bytes_used, _junk, _junk) = broker.get_info() line_data = '"%s",%d,%d,%d\n' % ( account_name, container_count, object_count, bytes_used) statfile.write(line_data) hasher.update(line_data) file_hash = hasher.hexdigest() hash_index = src_filename.find('*') if hash_index < 0: # if there is no * in the target filename, the uploader probably # won't work because we are crafting a filename that doesn't # fit the pattern src_filename = '_'.join([src_filename, file_hash]) else: parts = src_filename[:hash_index], src_filename[hash_index + 1:] src_filename = ''.join([parts[0], file_hash, parts[1]]) renamer(tmp_filename, os.path.join(self.target_dir, src_filename)) shutil.rmtree(working_dir, ignore_errors=True)
def account_audit(self, path): """ Audits the given account path :param path: the path to an account db """ try: if not path.endswith('.db'): return broker = AccountBroker(path) if not broker.is_deleted(): info = broker.get_info() self.account_passes += 1 self.logger.debug(_('Audit passed for %s') % broker.db_file) except Exception: self.account_failures += 1 self.logger.exception(_('ERROR Could not get account info %s'), (broker.db_file))
def reap_device(self, device): """ Called once per pass for each device on the server. This will scan the accounts directory for the device, looking for partitions this device is the primary for, then looking for account databases that are marked status=DELETED and still have containers and calling :func:`reap_account`. Account databases marked status=DELETED that no longer have containers will eventually be permanently removed by the reclaim process within the account replicator (see :mod:`swift.db_replicator`). :param device: The device to look for accounts to be deleted. """ datadir = os.path.join(self.devices, device, DATADIR) if not os.path.exists(datadir): return for partition in os.listdir(datadir): partition_path = os.path.join(datadir, partition) if not partition.isdigit(): continue nodes = self.get_account_ring().get_part_nodes(int(partition)) if nodes[0]['ip'] not in self.myips or \ not os.path.isdir(partition_path): continue for suffix in os.listdir(partition_path): suffix_path = os.path.join(partition_path, suffix) if not os.path.isdir(suffix_path): continue for hsh in os.listdir(suffix_path): hsh_path = os.path.join(suffix_path, hsh) if not os.path.isdir(hsh_path): continue for fname in sorted(os.listdir(hsh_path), reverse=True): if fname.endswith('.ts'): break elif fname.endswith('.db'): self.start_time = time() broker = \ AccountBroker(os.path.join(hsh_path, fname)) if broker.is_status_deleted() and \ not broker.empty(): self.reap_account(broker, partition, nodes)
def account_audit(self, path): """ Audits the given account path :param path: the path to an account db """ start_time = time.time() try: broker = AccountBroker(path) if not broker.is_deleted(): broker.get_info() self.logger.increment('passes') self.account_passes += 1 self.logger.debug(_('Audit passed for %s') % broker.db_file) except (Exception, Timeout): self.logger.increment('failures') self.account_failures += 1 self.logger.exception(_('ERROR Could not get account info %s'), (broker.db_file)) self.logger.timing_since('timing', start_time)