def find_and_process(self):
src_filename = time.strftime(self.filename_format)
working_dir = os.path.join(self.target_dir, '.stats_tmp')
shutil.rmtree(working_dir, ignore_errors=True)
mkdirs(working_dir)
tmp_filename = os.path.join(working_dir, src_filename)
hasher = hashlib.md5()
with open(tmp_filename, 'wb') as statfile:
# csv has the following columns:
# Account Name, Container Count, Object Count, Bytes Used
for device in os.listdir(self.devices):
if self.mount_check and not check_mount(self.devices, device):
self.logger.error(
_("Device %s is not mounted, skipping.") % device)
continue
accounts = os.path.join(self.devices,
device,
account_server_data_dir)
if not os.path.exists(accounts):
self.logger.debug(_("Path %s does not exist, skipping.") %
accounts)
continue
for root, dirs, files in os.walk(accounts, topdown=False):
for filename in files:
if filename.endswith('.db'):
db_path = os.path.join(root, filename)
broker = AccountBroker(db_path)
if not broker.is_deleted():
(account_name,
_junk, _junk, _junk,
container_count,
object_count,
bytes_used,
_junk, _junk) = broker.get_info()
line_data = '"%s",%d,%d,%d\n' % (
account_name, container_count,
object_count, bytes_used)
statfile.write(line_data)
hasher.update(line_data)
file_hash = hasher.hexdigest()
hash_index = src_filename.find('*')
if hash_index < 0:
# if there is no * in the target filename, the uploader probably
# won't work because we are crafting a filename that doesn't
# fit the pattern
src_filename = '_'.join([src_filename, file_hash])
else:
parts = src_filename[:hash_index], src_filename[hash_index + 1:]
src_filename = ''.join([parts[0], file_hash, parts[1]])
renamer(tmp_filename, os.path.join(self.target_dir, src_filename))
shutil.rmtree(working_dir, ignore_errors=True)
def get_data(self, db_path):
"""
Data for generated csv has the following columns:
Account Hash, Container Count, Object Count, Bytes Used
:raises sqlite3.Error: does not catch errors connecting to db
"""
line_data = None
broker = AccountBroker(db_path)
if not broker.is_deleted():
info = broker.get_info()
line_data = '"%s",%d,%d,%d\n' % (info['account'],
info['container_count'],
info['object_count'],
info['bytes_used'])
return line_data
def find_and_process(self):
src_filename = time.strftime(self.filename_format)
working_dir = os.path.join(self.target_dir, '.stats_tmp')
shutil.rmtree(working_dir, ignore_errors=True)
mkdirs(working_dir)
tmp_filename = os.path.join(working_dir, src_filename)
hasher = hashlib.md5()
with open(tmp_filename, 'wb') as statfile:
# csv has the following columns:
# Account Name, Container Count, Object Count, Bytes Used
for device in os.listdir(self.devices):
if self.mount_check and not check_mount(self.devices, device):
self.logger.error(
_("Device %s is not mounted, skipping.") % device)
continue
accounts = os.path.join(self.devices, device,
account_server_data_dir)
if not os.path.exists(accounts):
self.logger.debug(
_("Path %s does not exist, skipping.") % accounts)
continue
for root, dirs, files in os.walk(accounts, topdown=False):
for filename in files:
if filename.endswith('.db'):
db_path = os.path.join(root, filename)
broker = AccountBroker(db_path)
if not broker.is_deleted():
(account_name, _junk, _junk, _junk,
container_count, object_count, bytes_used,
_junk, _junk) = broker.get_info()
line_data = '"%s",%d,%d,%d\n' % (
account_name, container_count,
object_count, bytes_used)
statfile.write(line_data)
hasher.update(line_data)
file_hash = hasher.hexdigest()
hash_index = src_filename.find('*')
if hash_index < 0:
# if there is no * in the target filename, the uploader probably
# won't work because we are crafting a filename that doesn't
# fit the pattern
src_filename = '_'.join([src_filename, file_hash])
else:
parts = src_filename[:hash_index], src_filename[hash_index + 1:]
src_filename = ''.join([parts[0], file_hash, parts[1]])
renamer(tmp_filename, os.path.join(self.target_dir, src_filename))
shutil.rmtree(working_dir, ignore_errors=True)
def account_audit(self, path):
"""
Audits the given account path
:param path: the path to an account db
"""
try:
if not path.endswith(".db"):
return
broker = AccountBroker(path)
if not broker.is_deleted():
info = broker.get_info()
self.account_passes += 1
self.logger.debug(_("Audit passed for %s") % broker.db_file)
except Exception:
self.account_failures += 1
self.logger.exception(_("ERROR Could not get account info %s"), (broker.db_file))
def account_audit(self, path):
"""
Audits the given account path
:param path: the path to an account db
"""
try:
if not path.endswith('.db'):
return
broker = AccountBroker(path)
if not broker.is_deleted():
info = broker.get_info()
self.account_passes += 1
self.logger.debug(_('Audit passed for %s') % broker.db_file)
except Exception:
self.account_failures += 1
self.logger.exception(_('ERROR Could not get account info %s'),
(broker.db_file))
def account_audit(self, path):
"""
Audits the given account path
:param path: the path to an account db
"""
start_time = time.time()
try:
broker = AccountBroker(path)
if not broker.is_deleted():
broker.get_info()
self.logger.increment('passes')
self.account_passes += 1
self.logger.debug(_('Audit passed for %s') % broker.db_file)
except (Exception, Timeout):
self.logger.increment('failures')
self.account_failures += 1
self.logger.exception(_('ERROR Could not get account info %s'),
(broker.db_file))
self.logger.timing_since('timing', start_time)
def account_audit(self, path):
"""
Audits the given account path
:param path: the path to an account db
"""
start_time = time.time()
try:
broker = AccountBroker(path)
if not broker.is_deleted():
broker.get_info()
self.logger.increment('passes')
self.account_passes += 1
self.logger.debug(_('Audit passed for %s') % broker.db_file)
except (Exception, Timeout):
self.logger.increment('failures')
self.account_failures += 1
self.logger.exception(_('ERROR Could not get account info %s'),
(broker.db_file))
self.logger.timing_since('timing', start_time)
def account_audit(self, path):
"""
Audits the given account path
:param path: the path to an account db
"""
start_time = time.time()
try:
if not path.endswith(".db"):
return
broker = AccountBroker(path)
if not broker.is_deleted():
info = broker.get_info()
self.logger.increment("passes")
self.account_passes += 1
self.logger.debug(_("Audit passed for %s") % broker.db_file)
except (Exception, Timeout):
self.logger.increment("failures")
self.account_failures += 1
self.logger.exception(_("ERROR Could not get account info %s"), (broker.db_file))
self.logger.timing_since("timing", start_time)
