class ObjectExpirer(Daemon):
"""
Daemon that queries the internal hidden expiring_objects_account to
discover objects that need to be deleted.
:param conf: The daemon configuration.
"""
def __init__(self, conf):
self.conf = conf
self.logger = get_logger(conf, log_route='object-expirer')
self.interval = int(conf.get('interval') or 300)
self.expiring_objects_account = \
(conf.get('auto_create_account_prefix') or '.') + \
'expiring_objects'
conf_path = conf.get('__file__') or '/etc/swift/object-expirer.conf'
request_tries = int(conf.get('request_tries') or 3)
self.swift = InternalClient(conf_path,
'Swift Object Expirer',
request_tries)
self.report_interval = int(conf.get('report_interval') or 300)
self.report_first_time = self.report_last_time = time()
self.report_objects = 0
self.recon_cache_path = conf.get('recon_cache_path',
'/var/cache/swift')
self.rcache = join(self.recon_cache_path, 'object.recon')
self.concurrency = int(conf.get('concurrency', 1))
if self.concurrency < 1:
raise ValueError("concurrency must be set to at least 1")
self.processes = int(self.conf.get('processes', 0))
self.process = int(self.conf.get('process', 0))
def report(self, final=False):
"""
Emits a log line report of the progress so far, or the final progress
is final=True.
:param final: Set to True for the last report once the expiration pass
has completed.
"""
if final:
elapsed = time() - self.report_first_time
self.logger.info(_('Pass completed in %ds; %d objects expired') %
(elapsed, self.report_objects))
dump_recon_cache({'object_expiration_pass': elapsed,
'expired_last_pass': self.report_objects},
self.rcache, self.logger)
elif time() - self.report_last_time >= self.report_interval:
elapsed = time() - self.report_first_time
self.logger.info(_('Pass so far %ds; %d objects expired') %
(elapsed, self.report_objects))
self.report_last_time = time()
def run_once(self, *args, **kwargs):
"""
Executes a single pass, looking for objects to expire.
:param args: Extra args to fulfill the Daemon interface; this daemon
has no additional args.
:param kwargs: Extra keyword args to fulfill the Daemon interface; this
daemon accepts processes and process keyword args.
These will override the values from the config file if
provided.
"""
processes, process = self.get_process_values(kwargs)
pool = GreenPool(self.concurrency)
containers_to_delete = []
self.report_first_time = self.report_last_time = time()
self.report_objects = 0
try:
self.logger.debug(_('Run begin'))
containers, objects = \
self.swift.get_account_info(self.expiring_objects_account)
self.logger.info(_('Pass beginning; %s possible containers; %s '
'possible objects') % (containers, objects))
for c in self.swift.iter_containers(self.expiring_objects_account):
container = c['name']
timestamp = int(container)
if timestamp > int(time()):
break
containers_to_delete.append(container)
for o in self.swift.iter_objects(self.expiring_objects_account,
container):
obj = o['name'].encode('utf8')
if processes > 0:
obj_process = int(
hashlib.md5('%s/%s' % (container, obj)).
hexdigest(), 16)
if obj_process % processes != process:
continue
timestamp, actual_obj = obj.split('-', 1)
timestamp = int(timestamp)
if timestamp > int(time()):
break
pool.spawn_n(
self.delete_object, actual_obj, timestamp,
container, obj)
pool.waitall()
for container in containers_to_delete:
try:
self.swift.delete_container(
self.expiring_objects_account,
container,
acceptable_statuses=(2, HTTP_NOT_FOUND, HTTP_CONFLICT))
except (Exception, Timeout), err:
self.logger.exception(
_('Exception while deleting container %s %s') %
(container, str(err)))
self.logger.debug(_('Run end'))
self.report(final=True)
except (Exception, Timeout):
self.logger.exception(_('Unhandled exception'))
class ObjectExpirer(Daemon):
"""
Daemon that queries the internal hidden expiring_objects_account to
discover objects that need to be deleted.
:param conf: The daemon configuration.
"""
def __init__(self, conf):
self.conf = conf
self.logger = get_logger(conf, log_route='object-expirer')
self.interval = int(conf.get('interval') or 300)
self.expiring_objects_account = \
(conf.get('auto_create_account_prefix') or '.') + \
'expiring_objects'
conf_path = conf.get('__file__') or '/etc/swift/object-expirer.conf'
request_tries = int(conf.get('request_tries') or 3)
self.swift = InternalClient(conf_path, 'Swift Object Expirer',
request_tries)
self.report_interval = int(conf.get('report_interval') or 300)
self.report_first_time = self.report_last_time = time()
self.report_objects = 0
self.recon_cache_path = conf.get('recon_cache_path',
'/var/cache/swift')
self.rcache = join(self.recon_cache_path, 'object.recon')
self.concurrency = int(conf.get('concurrency', 1))
if self.concurrency < 1:
raise ValueError("concurrency must be set to at least 1")
self.processes = int(self.conf.get('processes', 0))
self.process = int(self.conf.get('process', 0))
def report(self, final=False):
"""
Emits a log line report of the progress so far, or the final progress
is final=True.
:param final: Set to True for the last report once the expiration pass
has completed.
"""
if final:
elapsed = time() - self.report_first_time
self.logger.info(
_('Pass completed in %ds; %d objects expired') %
(elapsed, self.report_objects))
dump_recon_cache(
{
'object_expiration_pass': elapsed,
'expired_last_pass': self.report_objects
}, self.rcache, self.logger)
elif time() - self.report_last_time >= self.report_interval:
elapsed = time() - self.report_first_time
self.logger.info(
_('Pass so far %ds; %d objects expired') %
(elapsed, self.report_objects))
self.report_last_time = time()
def run_once(self, *args, **kwargs):
"""
Executes a single pass, looking for objects to expire.
:param args: Extra args to fulfill the Daemon interface; this daemon
has no additional args.
:param kwargs: Extra keyword args to fulfill the Daemon interface; this
daemon accepts processes and process keyword args.
These will override the values from the config file if
provided.
"""
processes, process = self.get_process_values(kwargs)
pool = GreenPool(self.concurrency)
containers_to_delete = []
self.report_first_time = self.report_last_time = time()
self.report_objects = 0
try:
self.logger.debug(_('Run begin'))
containers, objects = \
self.swift.get_account_info(self.expiring_objects_account)
self.logger.info(
_('Pass beginning; %s possible containers; %s '
'possible objects') % (containers, objects))
for c in self.swift.iter_containers(self.expiring_objects_account):
container = c['name']
timestamp = int(container)
if timestamp > int(time()):
break
containers_to_delete.append(container)
for o in self.swift.iter_objects(self.expiring_objects_account,
container):
obj = o['name'].encode('utf8')
if processes > 0:
obj_process = int(
hashlib.md5('%s/%s' %
(container, obj)).hexdigest(), 16)
if obj_process % processes != process:
continue
timestamp, actual_obj = obj.split('-', 1)
timestamp = int(timestamp)
if timestamp > int(time()):
break
pool.spawn_n(self.delete_object, actual_obj, timestamp,
container, obj)
pool.waitall()
for container in containers_to_delete:
try:
self.swift.delete_container(
self.expiring_objects_account,
container,
acceptable_statuses=(2, HTTP_NOT_FOUND, HTTP_CONFLICT))
except (Exception, Timeout) as err:
self.logger.exception(
_('Exception while deleting container %s %s') %
(container, str(err)))
self.logger.debug(_('Run end'))
self.report(final=True)
except (Exception, Timeout):
self.logger.exception(_('Unhandled exception'))
def run_forever(self, *args, **kwargs):
"""
Executes passes forever, looking for objects to expire.
:param args: Extra args to fulfill the Daemon interface; this daemon
has no additional args.
:param kwargs: Extra keyword args to fulfill the Daemon interface; this
daemon has no additional keyword args.
"""
sleep(random() * self.interval)
while True:
begin = time()
try:
self.run_once(*args, **kwargs)
except (Exception, Timeout):
self.logger.exception(_('Unhandled exception'))
elapsed = time() - begin
if elapsed < self.interval:
sleep(random() * (self.interval - elapsed))
def get_process_values(self, kwargs):
"""
Gets the processes, process from the kwargs if those values exist.
Otherwise, return processes, process set in the config file.
:param kwargs: Keyword args passed into the run_forever(), run_once()
methods. They have values specified on the command
line when the daemon is run.
"""
if kwargs.get('processes') is not None:
processes = int(kwargs['processes'])
else:
processes = self.processes
if kwargs.get('process') is not None:
process = int(kwargs['process'])
else:
process = self.process
if process < 0:
raise ValueError(
'process must be an integer greater than or equal to 0')
if processes < 0:
raise ValueError(
'processes must be an integer greater than or equal to 0')
if processes and process >= processes:
raise ValueError('process must be less than or equal to processes')
return processes, process
def delete_object(self, actual_obj, timestamp, container, obj):
start_time = time()
try:
self.delete_actual_object(actual_obj, timestamp)
self.swift.delete_object(self.expiring_objects_account, container,
obj)
self.report_objects += 1
self.logger.increment('objects')
except (Exception, Timeout) as err:
self.logger.increment('errors')
self.logger.exception(
_('Exception while deleting object %s %s %s') %
(container, obj, str(err)))
self.logger.timing_since('timing', start_time)
self.report()
def delete_actual_object(self, actual_obj, timestamp):
"""
Deletes the end-user object indicated by the actual object name given
'<account>/<container>/<object>' if and only if the X-Delete-At value
of the object is exactly the timestamp given.
:param actual_obj: The name of the end-user object to delete:
'<account>/<container>/<object>'
:param timestamp: The timestamp the X-Delete-At value must match to
perform the actual delete.
"""
path = '/v1/' + urllib.quote(actual_obj.lstrip('/'))
self.swift.make_request('DELETE', path,
{'X-If-Delete-At': str(timestamp)},
(2, HTTP_NOT_FOUND, HTTP_PRECONDITION_FAILED))
class ObjectExpirer(Daemon):
"""
Daemon that queries the internal hidden expiring_objects_account to
discover objects that need to be deleted.
:param conf: The daemon configuration.
"""
def __init__(self, conf):
self.conf = conf
self.logger = get_logger(conf, log_route='object-expirer')
self.interval = int(conf.get('interval') or 300)
self.expiring_objects_account = \
(conf.get('auto_create_account_prefix') or '.') + \
'expiring_objects'
conf_path = conf.get('__file__') or '/etc/swift/object-expirer.conf'
request_tries = int(conf.get('request_tries') or 3)
self.swift = InternalClient(conf_path,
'Swift Object Expirer',
request_tries)
self.report_interval = int(conf.get('report_interval') or 300)
self.report_first_time = self.report_last_time = time()
self.report_objects = 0
self.recon_cache_path = conf.get('recon_cache_path',
'/var/cache/swift')
self.rcache = join(self.recon_cache_path, 'object.recon')
def report(self, final=False):
"""
Emits a log line report of the progress so far, or the final progress
is final=True.
:param final: Set to True for the last report once the expiration pass
has completed.
"""
if final:
elapsed = time() - self.report_first_time
self.logger.info(_('Pass completed in %ds; %d objects expired') %
(elapsed, self.report_objects))
dump_recon_cache({'object_expiration_pass': elapsed,
'expired_last_pass': self.report_objects},
self.rcache, self.logger)
elif time() - self.report_last_time >= self.report_interval:
elapsed = time() - self.report_first_time
self.logger.info(_('Pass so far %ds; %d objects expired') %
(elapsed, self.report_objects))
self.report_last_time = time()
def run_once(self, *args, **kwargs):
"""
Executes a single pass, looking for objects to expire.
:param args: Extra args to fulfill the Daemon interface; this daemon
has no additional args.
:param kwargs: Extra keyword args to fulfill the Daemon interface; this
daemon has no additional keyword args.
"""
self.report_first_time = self.report_last_time = time()
self.report_objects = 0
try:
self.logger.debug(_('Run begin'))
containers, objects = \
self.swift.get_account_info(self.expiring_objects_account)
self.logger.info(_('Pass beginning; %s possible containers; %s '
'possible objects') % (containers, objects))
for c in self.swift.iter_containers(self.expiring_objects_account):
container = c['name']
timestamp = int(container)
if timestamp > int(time()):
break
for o in self.swift.iter_objects(self.expiring_objects_account,
container):
obj = o['name']
timestamp, actual_obj = obj.split('-', 1)
timestamp = int(timestamp)
if timestamp > int(time()):
break
start_time = time()
try:
self.delete_actual_object(actual_obj, timestamp)
self.swift.delete_object(self.expiring_objects_account,
container, obj)
self.report_objects += 1
self.logger.increment('objects')
except (Exception, Timeout), err:
self.logger.increment('errors')
self.logger.exception(
_('Exception while deleting object %s %s %s') %
(container, obj, str(err)))
self.logger.timing_since('timing', start_time)
self.report()
try:
self.swift.delete_container(
self.expiring_objects_account,
container,
acceptable_statuses=(2, HTTP_NOT_FOUND, HTTP_CONFLICT))
except (Exception, Timeout), err:
self.logger.exception(
_('Exception while deleting container %s %s') %
(container, str(err)))
self.logger.debug(_('Run end'))
self.report(final=True)
class ObjectExpirer(Daemon):
"""
Daemon that queries the internal hidden expiring_objects_account to
discover objects that need to be deleted.
:param conf: The daemon configuration.
"""
def __init__(self, conf):
self.conf = conf
self.logger = get_logger(conf, log_route='object-expirer')
self.interval = int(conf.get('interval') or 300)
self.expiring_objects_account = \
(conf.get('auto_create_account_prefix') or '.') + \
(conf.get('expiring_objects_account_name') or 'expiring_objects')
conf_path = conf.get('__file__') or '/etc/swift/object-expirer.conf'
request_tries = int(conf.get('request_tries') or 3)
self.swift = InternalClient(conf_path,
'Swift Object Expirer',
request_tries)
self.report_interval = int(conf.get('report_interval') or 300)
self.report_first_time = self.report_last_time = time()
self.report_objects = 0
self.recon_cache_path = conf.get('recon_cache_path',
'/var/cache/swift')
self.rcache = join(self.recon_cache_path, 'object.recon')
self.concurrency = int(conf.get('concurrency', 1))
if self.concurrency < 1:
raise ValueError("concurrency must be set to at least 1")
self.processes = int(self.conf.get('processes', 0))
self.process = int(self.conf.get('process', 0))
def report(self, final=False):
"""
Emits a log line report of the progress so far, or the final progress
is final=True.
:param final: Set to True for the last report once the expiration pass
has completed.
"""
if final:
elapsed = time() - self.report_first_time
self.logger.info(_('Pass completed in %ds; %d objects expired') %
(elapsed, self.report_objects))
dump_recon_cache({'object_expiration_pass': elapsed,
'expired_last_pass': self.report_objects},
self.rcache, self.logger)
elif time() - self.report_last_time >= self.report_interval:
elapsed = time() - self.report_first_time
self.logger.info(_('Pass so far %ds; %d objects expired') %
(elapsed, self.report_objects))
self.report_last_time = time()
def run_once(self, *args, **kwargs):
"""
Executes a single pass, looking for objects to expire.
:param args: Extra args to fulfill the Daemon interface; this daemon
has no additional args.
:param kwargs: Extra keyword args to fulfill the Daemon interface; this
daemon accepts processes and process keyword args.
These will override the values from the config file if
provided.
"""
processes, process = self.get_process_values(kwargs)
pool = GreenPool(self.concurrency)
containers_to_delete = []
self.report_first_time = self.report_last_time = time()
self.report_objects = 0
try:
self.logger.debug(_('Run begin'))
containers, objects = \
self.swift.get_account_info(self.expiring_objects_account)
self.logger.info(_('Pass beginning; %s possible containers; %s '
'possible objects') % (containers, objects))
for c in self.swift.iter_containers(self.expiring_objects_account):
container = c['name']
timestamp = int(container)
if timestamp > int(time()):
break
containers_to_delete.append(container)
for o in self.swift.iter_objects(self.expiring_objects_account,
container):
obj = o['name'].encode('utf8')
if processes > 0:
obj_process = int(
hashlib.md5('%s/%s' % (container, obj)).
hexdigest(), 16)
if obj_process % processes != process:
continue
timestamp, actual_obj = obj.split('-', 1)
timestamp = int(timestamp)
if timestamp > int(time()):
break
pool.spawn_n(
self.delete_object, actual_obj, timestamp,
container, obj)
pool.waitall()
for container in containers_to_delete:
try:
self.swift.delete_container(
self.expiring_objects_account,
container,
acceptable_statuses=(2, HTTP_NOT_FOUND, HTTP_CONFLICT))
except (Exception, Timeout) as err:
self.logger.exception(
_('Exception while deleting container %s %s') %
(container, str(err)))
self.logger.debug(_('Run end'))
self.report(final=True)
except (Exception, Timeout):
self.logger.exception(_('Unhandled exception'))
def run_forever(self, *args, **kwargs):
"""
Executes passes forever, looking for objects to expire.
:param args: Extra args to fulfill the Daemon interface; this daemon
has no additional args.
:param kwargs: Extra keyword args to fulfill the Daemon interface; this
daemon has no additional keyword args.
"""
sleep(random() * self.interval)
while True:
begin = time()
try:
self.run_once(*args, **kwargs)
except (Exception, Timeout):
self.logger.exception(_('Unhandled exception'))
elapsed = time() - begin
if elapsed < self.interval:
sleep(random() * (self.interval - elapsed))
def get_process_values(self, kwargs):
"""
Gets the processes, process from the kwargs if those values exist.
Otherwise, return processes, process set in the config file.
:param kwargs: Keyword args passed into the run_forever(), run_once()
methods. They have values specified on the command
line when the daemon is run.
"""
if kwargs.get('processes') is not None:
processes = int(kwargs['processes'])
else:
processes = self.processes
if kwargs.get('process') is not None:
process = int(kwargs['process'])
else:
process = self.process
if process < 0:
raise ValueError(
'process must be an integer greater than or equal to 0')
if processes < 0:
raise ValueError(
'processes must be an integer greater than or equal to 0')
if processes and process >= processes:
raise ValueError(
'process must be less than or equal to processes')
return processes, process
def delete_object(self, actual_obj, timestamp, container, obj):
start_time = time()
try:
self.delete_actual_object(actual_obj, timestamp)
self.swift.delete_object(self.expiring_objects_account,
container, obj)
self.report_objects += 1
self.logger.increment('objects')
except (Exception, Timeout) as err:
self.logger.increment('errors')
self.logger.exception(
_('Exception while deleting object %s %s %s') %
(container, obj, str(err)))
self.logger.timing_since('timing', start_time)
self.report()
def delete_actual_object(self, actual_obj, timestamp):
"""
Deletes the end-user object indicated by the actual object name given
'<account>/<container>/<object>' if and only if the X-Delete-At value
of the object is exactly the timestamp given.
:param actual_obj: The name of the end-user object to delete:
'<account>/<container>/<object>'
:param timestamp: The timestamp the X-Delete-At value must match to
perform the actual delete.
"""
path = '/v1/' + urllib.quote(actual_obj.lstrip('/'))
self.swift.make_request('DELETE', path,
{'X-If-Delete-At': str(timestamp)},
(2, HTTP_NOT_FOUND, HTTP_PRECONDITION_FAILED))
class UtilizationAggregator(Daemon):
def __init__(self, conf):
self.conf = conf
self.logger = get_logger(conf, log_route='utilization-aggregator')
self.interval = int(conf.get('interval') or 60)
self.aggregate_account = '.utilization'
self.sample_account = '.transfer_record'
conf_path = conf.get('__file__') or \
'/etc/swift/swift-utilization-aggregator.conf'
request_tries = int(conf.get('request_tries') or 3)
self.swift = InternalClient(conf_path,
'Swift Utilization Aggregator',
request_tries)
self.report_interval = int(conf.get('report_interval') or 60)
self.report_first_time = self.report_last_time = time()
self.report_containers = 0
self.report_objects = 0
self.recon_cache_path = conf.get('recon_cache_path',
'/var/cache/swift')
self.rcache = join(self.recon_cache_path, 'object.recon')
self.concurrency = int(conf.get('concurrency', 1))
if self.concurrency < 1:
raise ValueError("concurrency must be set to at least 1")
self.processes = int(self.conf.get('processes', 0))
self.process = int(self.conf.get('process', 0))
self.container_ring = Ring('/etc/swift', ring_name='container')
self.sample_rate = int(self.conf.get('sample_rate', 600))
self.last_chk = iso8601_to_timestamp(self.conf.get(
'service_start'))
self.kinx_api_url = self.conf.get('kinx_api_url')
def report(self, final=False):
if final:
elapsed = time() - self.report_first_time
self.logger.info(_('Pass completed in %ds; %d containers,'
' %d objects aggregated') %
(elapsed, self.report_containers,
self.report_objects))
dump_recon_cache({'object_aggregation_pass': elapsed,
'aggregation_last_pass': self.report_containers},
self.rcache, self.logger)
elif time() - self.report_last_time >= self.report_interval:
elapsed = time() - self.report_first_time
self.logger.info(_('Pass so far %ds; %d objects aggregated') %
(elapsed, self.report_objects))
self.report_last_time = time()
def run_once(self, *args, **kwargs):
processes, process = self.get_process_values(kwargs)
pool = GreenPool(self.concurrency)
self.report_first_time = self.report_last_time = time()
self.report_objects = 0
self.report_containers = 0
containers_to_delete = []
try:
self.logger.debug(_('Run begin'))
containers, objects = \
self.swift.get_account_info(self.sample_account)
self.logger.info(_('Pass beginning; %s possible containers; %s '
'possible objects') % (containers, objects))
for c in self.swift.iter_containers(self.sample_account):
container = c['name']
try:
timestamp, account = container.split('_', 1)
timestamp = float(timestamp)
except ValueError:
self.logger.debug('ValueError: %s, '
'need more than 1 value to unpack' % \
container)
else:
if processes > 0:
obj_proc = int(hashlib.md5(container).hexdigest(), 16)
if obj_proc % processes != process:
continue
n = (float(time()) // self.sample_rate) * self.sample_rate
if timestamp <= n:
containers_to_delete.append(container)
pool.spawn_n(self.aggregate_container, container)
pool.waitall()
for container in containers_to_delete:
try:
self.logger.debug('delete container: %s' % container)
self.swift.delete_container(self.sample_account, container,
acceptable_statuses=(
2, HTTP_NOT_FOUND,
HTTP_CONFLICT))
except (Exception, Timeout) as err:
self.logger.exception(
_('Exception while deleting container %s %s') %
(container, str(err)))
tenants_to_fillup = list()
for c in self.swift.iter_containers(self.aggregate_account):
tenant_id = c['name']
if processes > 0:
c_proc = int(hashlib.md5(tenant_id).hexdigest(), 16)
if c_proc % processes != process:
continue
tenants_to_fillup.append(tenant_id)
# fillup lossed usage data
self.fillup_lossed_usage_data(tenants_to_fillup)
self.logger.debug(_('Run end'))
self.report(final=True)
except (Exception, Timeout):
self.logger.exception(_('Unhandled exception'))
def run_forever(self, *args, **kwargs):
"""
Executes passes forever, looking for objects to expire.
:param args: Extra args to fulfill the Daemon interface; this daemon
has no additional args.
:param kwargs: Extra keyword args to fulfill the Daemon interface; this
daemon has no additional keyword args.
"""
sleep(random() * self.interval)
while True:
begin = time()
try:
self.run_once(*args, **kwargs)
except (Exception, Timeout):
self.logger.exception(_('Unhandled exception'))
elapsed = time() - begin
if elapsed < self.interval:
sleep(random() * (self.interval - elapsed))
def get_process_values(self, kwargs):
"""
Gets the processes, process from the kwargs if those values exist.
Otherwise, return processes, process set in the config file.
:param kwargs: Keyword args passed into the run_forever(), run_once()
methods. They have values specified on the command
line when the daemon is run.
"""
if kwargs.get('processes') is not None:
processes = int(kwargs['processes'])
else:
processes = self.processes
if kwargs.get('process') is not None:
process = int(kwargs['process'])
else:
process = self.process
if process < 0:
raise ValueError(
'process must be an integer greater than or equal to 0')
if processes < 0:
raise ValueError(
'processes must be an integer greater than or equal to 0')
if processes and process >= processes:
raise ValueError(
'process must be less than or equal to processes')
return processes, process
def aggregate_container(self, container):
start_time = time()
try:
objs_to_delete = list()
bytes_recvs = dict()
bytes_sents = dict()
ts, tenant_id, account = container.split('_', 2)
ts = int(float(ts))
for o in self.swift.iter_objects(self.sample_account, container):
name = o['name']
objs_to_delete.append(name)
ts, bytes_rv, bytes_st, trans_id, client_ip = name.split('/')
bill_type = self.get_billtype_by_client_ip(client_ip, ts)
bytes_recvs[bill_type] = bytes_recvs.get(bill_type,
0) + int(bytes_rv)
bytes_sents[bill_type] = bytes_sents.get(bill_type,
0) + int(bytes_st)
self.report_objects += 1
for o in objs_to_delete:
self.swift.delete_object(self.sample_account, container, o)
for bill_type, bt_rv in bytes_recvs.items():
t_object = 'transfer/%d/%d/%d_%d_%d' % (ts, bill_type, bt_rv,
bytes_sents[bill_type],
self.report_objects)
self._hidden_update(tenant_id, t_object)
except (Exception, Timeout) as err:
self.logger.increment('errors')
self.logger.exception(
_('Exception while aggregating sample %s %s') %
(container, str(err)))
self.logger.timing_since('timing', start_time)
self.report()
def account_info(self, tenant_id, timestamp):
path = '/v1/%s/%s?prefix=usage/%d&limit=1' % (self.aggregate_account,
tenant_id, timestamp)
resp = self.swift.make_request('GET', path, {}, (2,))
if len(resp.body) == 0:
return 0, 0, 0
usages = resp.body.split('/', 2)[2].rstrip()
cont_cnt, obj_cnt, bt_used = usages.split('_')
return int(cont_cnt), int(obj_cnt), int(bt_used)
def _hidden_update(self, container, obj, method='PUT'):
hidden_path = '/%s/%s/%s' % (self.aggregate_account, container, obj)
part, nodes = self.container_ring.get_nodes(self.aggregate_account,
container)
for node in nodes:
ip = node['ip']
port = node['port']
dev = node['device']
action_headers = dict()
action_headers['user-agent'] = 'aggregator'
action_headers['X-Timestamp'] = normalize_timestamp(time())
action_headers['referer'] = 'aggregator-daemon'
action_headers['x-size'] = '0'
action_headers['x-content-type'] = "text/plain"
action_headers['x-etag'] = 'd41d8cd98f00b204e9800998ecf8427e'
conn = http_connect(ip, port, dev, part, method, hidden_path,
action_headers)
response = conn.getresponse()
response.read()
def fillup_lossed_usage_data(self, tenants):
now = (float(time()) // self.sample_rate) * self.sample_rate
path = '/v1/%s/%s?prefix=usage/%d&limit=1'
for t in tenants:
last = self.last_chk
cont_cnt = obj_cnt = bt_used = -1
while last <= now:
p = path % (self.aggregate_account, t, last)
resp = self.swift.make_request('GET', p, {}, (2,))
if len(resp.body) != 0:
usages = resp.body.split('/', 2)[2].rstrip()
c, o, bt = usages.split('_')
cont_cnt = int(c)
obj_cnt = int(o)
bt_used = int(bt)
else:
before = last - self.sample_rate
if cont_cnt == -1:
cont_cnt, obj_cnt, bt_used = \
self.account_info(self.aggregate_account, before)
obj = 'usage/%d/%d_%d_%d' % (last, cont_cnt, obj_cnt,
bt_used)
self._hidden_update(t, obj)
last += self.sample_rate
self.last_chk = now
def get_billtype_by_client_ip(self, client_ip, timestamp):
end_ts = timestamp_to_iso8601(timestamp + self.sample_rate - 1)
start_ts = timestamp_to_iso8601(timestamp)
params = {'start': start_ts, 'end': end_ts}
path = self.kinx_api_url + '/?%s' % (urllib.urlencode(params))
data = json.loads(urllib.urlopen(path).read())
bill_type = -1
for r in data['ip_ranges']:
bill_type = r['bill_type']
for cidr in r['ip_range']:
if self.ip_in_cidr(client_ip, cidr):
return bill_type
return bill_type
def ip_in_cidr(self, client_ip, cidr):
bt_to_bits = lambda b: bin(int(b))[2:].rjust(8, '0')
ip_to_bits = lambda ip: ''.join([bt_to_bits(b) for b in ip.split('.')])
client_ip_bits = ip_to_bits(client_ip)
ip, snet = cidr.split('/')
ip_bits = ip_to_bits(ip)
if client_ip_bits[:int(snet)] == ip_bits[:int(snet)]:
return True
else:
return False
class ObjectExpirer(Daemon):
def __init__(self, conf):
super(ObjectExpirer, self).__init__(conf)
self.conf = conf
self.logger = get_logger(conf, log_route='s3-object-expirer')
self.logger.set_statsd_prefix('s3-object-expirer')
self.interval = int(conf.get('interval') or 300)
self.s3_expiring_objects_account = \
(conf.get('auto_create_account_prefix') or '.') + \
(conf.get('expiring_objects_account_name') or
's3_expiring_objects')
conf_path = conf.get('__file__') or '/etc/swift/s3-object-expirer.conf'
request_tries = int(conf.get('request_tries') or 3)
self.swift = InternalClient(conf_path,
'Swift Object Expirer',
request_tries)
self.glacier = self._init_glacier()
self.glacier_account_prefix = '.glacier_'
self.report_interval = int(conf.get('report_interval') or 300)
self.report_first_time = self.report_last_time = time()
self.report_objects = 0
self.recon_cache_path = conf.get('recon_cache_path',
'/var/cache/swift')
self.rcache = join(self.recon_cache_path, 'object.recon')
self.concurrency = int(conf.get('concurrency', 1))
if self.concurrency < 1:
raise ValueError("concurrency must be set to at least 1")
self.processes = int(self.conf.get('processes', 0))
self.process = int(self.conf.get('process', 0))
self.client = Client(self.conf.get('sentry_sdn', ''))
def _init_glacier(self):
con = Layer2(region_name='ap-northeast-1')
return con.get_vault('swift-s3-transition')
def report(self, final=False):
"""
Emits a log line report of the progress so far, or the final progress
is final=True.
:param final: Set to True for the last report once the expiration pass
has completed.
"""
if final:
elapsed = time() - self.report_first_time
self.logger.info(_('Pass completed in %ds; %d objects expired') %
(elapsed, self.report_objects))
dump_recon_cache({'object_expiration_pass': elapsed,
'expired_last_pass': self.report_objects},
self.rcache, self.logger)
elif time() - self.report_last_time >= self.report_interval:
elapsed = time() - self.report_first_time
self.logger.info(_('Pass so far %ds; %d objects expired') %
(elapsed, self.report_objects))
self.report_last_time = time()
def run_once(self, *args, **kwargs):
"""
Executes a single pass, looking for objects to expire.
:param args: Extra args to fulfill the Daemon interface; this daemon
has no additional args.
:param kwargs: Extra keyword args to fulfill the Daemon interface; this
daemon accepts processes and process keyword args.
These will override the values from the config file if
provided.
"""
processes, process = self.get_process_values(kwargs)
pool = GreenPool(self.concurrency)
containers_to_delete = []
self.report_first_time = self.report_last_time = time()
self.report_objects = 0
try:
self.logger.debug(_('Run begin'))
containers, objects = \
self.swift.get_account_info(self.s3_expiring_objects_account)
self.logger.info(_('Pass beginning; %s possible containers; %s '
'possible objects') % (containers, objects))
for c in self.swift.iter_containers(self.
s3_expiring_objects_account):
container = c['name']
timestamp = int(container)
if timestamp > int(time()):
break
containers_to_delete.append(container)
for o in self.swift.iter_objects(self
.s3_expiring_objects_account,
container):
obj = o['name'].encode('utf8')
if processes > 0:
obj_process = int(
hashlib.md5('%s/%s' % (container, obj)).
hexdigest(), 16)
if obj_process % processes != process:
continue
pool.spawn_n(self.delete_object, container, obj)
pool.waitall()
for container in containers_to_delete:
try:
self.swift.delete_container(
self.s3_expiring_objects_account,
container,
acceptable_statuses=(2, HTTP_NOT_FOUND, HTTP_CONFLICT))
except (Exception, Timeout) as err:
report_exception(self.logger,
_('Exception while deleting container %s %s') %
(container, str(err)), self.client.captureException())
self.logger.debug(_('Run end'))
self.report(final=True)
except (Exception, Timeout):
report_exception(self.logger, _('Unhandled exception'), self.client)
def run_forever(self, *args, **kwargs):
"""
Executes passes forever, looking for objects to expire.
:param args: Extra args to fulfill the Daemon interface; this daemon
has no additional args.
:param kwargs: Extra keyword args to fulfill the Daemon interface; this
daemon has no additional keyword args.
"""
sleep(random() * self.interval)
while True:
begin = time()
try:
self.run_once(*args, **kwargs)
except (Exception, Timeout):
report_exception(self.logger, _('Unhandled exception'), self.client)
elapsed = time() - begin
if elapsed < self.interval:
sleep(random() * (self.interval - elapsed))
def get_process_values(self, kwargs):
"""
Gets the processes, process from the kwargs if those values exist.
Otherwise, return processes, process set in the config file.
:param kwargs: Keyword args passed into the run_forever(), run_once()
methods. They have values specified on the command
line when the daemon is run.
"""
if kwargs.get('processes') is not None:
processes = int(kwargs['processes'])
else:
processes = self.processes
if kwargs.get('process') is not None:
process = int(kwargs['process'])
else:
process = self.process
if process < 0:
raise ValueError(
'process must be an integer greater than or equal to 0')
if processes < 0:
raise ValueError(
'processes must be an integer greater than or equal to 0')
if processes and process >= processes:
raise ValueError(
'process must be less than or equal to processes')
return processes, process
def delete_object(self, hidden_container, obj):
start_time = time()
try:
account, container, object = obj.split('/', 2)
lifecycle = Lifecycle(account, container, object,
swift_client=self.swift)
object_header = lifecycle.object.headers
object_rule = lifecycle.get_object_rule_by_action('Expiration')
last_modified = gmt_to_timestamp(object_header['Last-Modified'])
validation_flg = lifecycle.object_lifecycle_validation()
if (validation_flg == LIFECYCLE_OK) or \
(validation_flg == DISABLED_TRANSITION):
times = calc_when_actions_do(object_rule, last_modified)
actual_expire_time = int(times['Expiration'])
if actual_expire_time == int(hidden_container):
self.delete_actual_object(obj)
if lifecycle.get_s3_storage_class() == 'GLACIER':
self.delete_glacier_object(obj)
self.report_objects += 1
self.logger.increment('objects')
self.swift.delete_object(self.s3_expiring_objects_account,
hidden_container, obj)
except (Exception, Timeout) as err:
self.logger.increment('errors')
report_exception(self.logger,
_('Exception while deleting object %s %s %s') %
(hidden_container, obj, str(err)), self.client)
self.logger.timing_since('timing', start_time)
self.report()
def delete_glacier_object(self, obj):
account, container, prefix = obj.split('/', 2)
glacier_hidden_account = self.glacier_account_prefix + account
objs = get_objects_by_prefix(glacier_hidden_account, container, prefix,
swift_client=self.swift)
glacier_obj = None
for o in objs:
name = get_glacier_objname_from_hidden_object(o)
if name == prefix:
glacier_obj = o
break
glacier_archive_id = get_glacier_key_from_hidden_object(glacier_obj)
self.glacier.delete_archive(glacier_archive_id)
self.swift.delete_object(glacier_hidden_account, container,
glacier_obj)
def delete_actual_object(self, obj):
"""
Deletes the end-user object indicated by the actual object name given
'<account>/<container>/<object>' if and only if the X-Delete-At value
of the object is exactly the timestamp given.
:param obj: The name of the end-user object to delete:
'<account>/<container>/<object>'
"""
path = '/v1/' + urllib.quote(obj.lstrip('/'))
self.swift.make_request('DELETE', path,
{}, (2, HTTP_NOT_FOUND))
class ObjectTransitor(Daemon):
def __init__(self, conf):
super(ObjectTransitor, self).__init__(conf)
self.conf = conf
self.logger = get_logger(conf, log_route='s3-object-transitor')
self.logger.set_statsd_prefix('s3-object-transitor')
self.interval = int(conf.get('interval') or 300)
self.s3_tr_objects_account = \
(conf.get('auto_create_account_prefix') or '.') + \
(conf.get('expiring_objects_account_name') or
's3_transitioning_objects')
conf_path = conf.get('__file__') or \
'/etc/swift/s3-object-transitor.conf'
request_tries = int(conf.get('request_tries') or 3)
self.swift = InternalClient(conf_path, 'Swift Object Transitor',
request_tries)
self.report_interval = int(conf.get('report_interval') or 300)
self.report_first_time = self.report_last_time = time()
self.report_objects = 0
self.recon_cache_path = conf.get('recon_cache_path',
'/var/cache/swift')
self.rcache = join(self.recon_cache_path, 'object.recon')
self.concurrency = int(conf.get('concurrency', 1))
if self.concurrency < 1:
raise ValueError("concurrency must be set to at least 1")
self.processes = int(self.conf.get('processes', 0))
self.process = int(self.conf.get('process', 0))
self.client = Client(self.conf.get('sentry_sdn', ''))
def report(self, final=False):
"""
Emits a log line report of the progress so far, or the final progress
is final=True.
:param final: Set to True for the last report once the expiration pass
has completed.
"""
if final:
elapsed = time() - self.report_first_time
self.logger.info(_('Pass completed in %ds; %d objects '
'transitioned') %
(elapsed, self.report_objects))
dump_recon_cache({'object_transition_pass': elapsed,
'transitioned_last_pass': self.report_objects},
self.rcache, self.logger)
elif time() - self.report_last_time >= self.report_interval:
elapsed = time() - self.report_first_time
self.logger.info(_('Pass so far %ds; %d objects transitioned') %
(elapsed, self.report_objects))
self.report_last_time = time()
def run_once(self, *args, **kwargs):
"""
Executes a single pass, looking for objects to expire.
:param args: Extra args to fulfill the Daemon interface; this daemon
has no additional args.
:param kwargs: Extra keyword args to fulfill the Daemon interface; this
daemon accepts processes and process keyword args.
These will override the values from the config file if
provided.
"""
processes, process = self.get_process_values(kwargs)
pool = GreenPool(self.concurrency)
containers_to_delete = []
self.report_first_time = self.report_last_time = time()
self.report_objects = 0
try:
self.logger.debug(_('Run begin'))
containers, objects = \
self.swift.get_account_info(self.s3_tr_objects_account)
self.logger.info(_('Pass beginning; %s possible containers; %s '
'possible objects') % (containers, objects))
for c in self.swift.iter_containers(self.s3_tr_objects_account):
container = c['name']
timestamp = int(container)
if timestamp > int(time()):
break
containers_to_delete.append(container)
for o in self.swift.iter_objects(self.s3_tr_objects_account,
container):
obj = o['name'].encode('utf8')
if processes > 0:
obj_process = int(
hashlib.md5('%s/%s' % (container, obj)).
hexdigest(), 16)
if obj_process % processes != process:
continue
pool.spawn_n(self.transition_object, container, obj)
pool.waitall()
for container in containers_to_delete:
try:
self.swift.delete_container(self.s3_tr_objects_account,
container, (2, 4))
except (Exception, Timeout) as err:
report_exception(self.logger,
_('Exception while deleting container %s %s') %
(container, str(err)), self.client)
self.logger.debug(_('Run end'))
self.report(final=True)
except (Exception, Timeout):
report_exception(self.logger, _('Unhandled exception'), self.client)
def run_forever(self, *args, **kwargs):
"""
Executes passes forever, looking for objects to expire.
:param args: Extra args to fulfill the Daemon interface; this daemon
has no additional args.
:param kwargs: Extra keyword args to fulfill the Daemon interface; this
daemon has no additional keyword args.
"""
sleep(random() * self.interval)
while True:
begin = time()
try:
self.run_once(*args, **kwargs)
except (Exception, Timeout):
report_exception(self.logger, _('Unhandled exception'), self.client)
elapsed = time() - begin
if elapsed < self.interval:
sleep(random() * (self.interval - elapsed))
def get_process_values(self, kwargs):
"""
Gets the processes, process from the kwargs if those values exist.
Otherwise, return processes, process set in the config file.
:param kwargs: Keyword args passed into the run_forever(), run_once()
methods. They have values specified on the command
line when the daemon is run.
"""
if kwargs.get('processes') is not None:
processes = int(kwargs['processes'])
else:
processes = self.processes
if kwargs.get('process') is not None:
process = int(kwargs['process'])
else:
process = self.process
if process < 0:
raise ValueError(
'process must be an integer greater than or equal to 0')
if processes < 0:
raise ValueError(
'processes must be an integer greater than or equal to 0')
if processes and process >= processes:
raise ValueError(
'process must be less than or equal to processes')
return processes, process
def transition_object(self, container, obj):
start_time = time()
try:
obj_account, obj_container, obj_object = obj.split('/', 2)
lifecycle = Lifecycle(obj_account, obj_container, obj_object,
swift_client=self.swift)
if is_success(lifecycle.object.status):
object_header = lifecycle.object.headers
object_rule = lifecycle.get_object_rule_by_action(
'Transition')
last_modified = object_header['Last-Modified']
last_modified = gmt_to_timestamp(last_modified)
validation_flg = lifecycle.object_lifecycle_validation()
if (validation_flg == LIFECYCLE_OK) or \
(validation_flg == DISABLED_EXPIRATION):
times = calc_when_actions_do(object_rule, last_modified)
actual_expire_time = int(times['Transition'])
if actual_expire_time == int(container):
self.request_transition(obj)
self.swift.delete_object(self.s3_tr_objects_account,
container, obj)
except (Exception, Timeout) as err:
self.logger.increment('errors')
report_exception(self.logger,
_('Exception while transitioning object %s %s %s') %
(container, obj, str(err)), self.client)
self.logger.timing_since('timing', start_time)
self.report()
def request_transition(self, actual_obj):
path = '/v1/' + urllib.quote(actual_obj.lstrip('/'))
headers = {GLACIER_FLAG_META: True,
'X-S3-Object-Transition': True}
resp = self.swift.make_request('POST', path, headers, (2, 5))
if resp.status_int == 500:
raise Exception(resp.body)
self.report_objects += 1
self.logger.increment('objects')
