def __init__(self, conf, logger, volume):
self.conf = conf
self.logger = logger or get_logger(conf)
self.volume = volume
self.run_time = 0
self.passes = 0
self.errors = 0
self.last_reported = 0
self.last_usage_check = 0
self.chunks_run_time = 0
self.bytes_running_time = 0
self.bytes_processed = 0
self.total_bytes_processed = 0
self.total_chunks_processed = 0
self.usage_target = int_value(
conf.get('usage_target'), 0)
self.usage_check_interval = int_value(
conf.get('usage_check_interval'), 3600)
self.report_interval = int_value(
conf.get('report_interval'), 3600)
self.max_chunks_per_second = int_value(
conf.get('chunks_per_second'), 30)
self.max_bytes_per_second = int_value(
conf.get('bytes_per_second'), 10000000)
self.blob_client = BlobClient()
self.container_client = ContainerClient(conf)
def __init__(self, conf, logger, volume):
self.conf = conf
self.logger = logger or get_logger(conf)
self.volume = volume
self.run_time = 0
self.passes = 0
self.errors = 0
self.last_reported = 0
self.chunks_run_time = 0
self.bytes_running_time = 0
self.bytes_processed = 0
self.total_bytes_processed = 0
self.total_chunks_processed = 0
self.dry_run = true_value(
conf.get('dry_run', False))
self.report_interval = int_value(
conf.get('report_interval'), 3600)
self.max_chunks_per_second = int_value(
conf.get('chunks_per_second'), 30)
self.max_bytes_per_second = int_value(
conf.get('bytes_per_second'), 10000000)
self.rdir_fetch_limit = int_value(
conf.get('rdir_fetch_limit'), 100)
self.rdir_client = RdirClient(conf)
self.content_factory = ContentFactory(conf)
def __init__(self, conf, service, **kwargs):
self.conf = conf
self.running = False
for k in ['host', 'port', 'type']:
if k not in service:
raise Exception(
'Missing field "%s" in service configuration' % k)
self.name = '%s|%s' % \
(service['host'], service['port'])
self.check_interval = float_value(conf.get('check_interval'), 1)
self.service = service
self.rise = int_value(conf.get('rise'), 1)
self.fall = int_value(conf.get('fall'), 1)
self.logger = get_logger(self.conf)
self.cs = ConscienceClient(self.conf)
self.client = Client(self.conf)
self.last_status = False
self.failed = False
self.service_definition = {
'ns': self.conf['namespace'],
'type': self.service['type'],
'addr': '%s:%s' % (self.service['host'], self.service['port']),
'score': 0,
'tags': {}}
if self.service.get('location', None):
self.service_definition['tags']['tag.loc'] = \
self.service['location']
self.service_checks = list()
self.service_stats = list()
self.init_checkers(service)
self.init_stats(service)
def __init__(self, conf):
validate_service_conf(conf)
self.conf = conf
self.logger = get_logger(conf)
self.running = False
self.retries_run_time = 0
self.max_retries_per_second = int_value(
conf.get('retries_per_second'), 30)
self.batch_size = int_value(conf.get('batch_size'), 500)
self.init_zmq()
self.init_queue()
self.init_workers()
def __init__(self, conf, logger, volume):
self.conf = conf
self.logger = logger
self.volume = volume
self.passes = 0
self.errors = 0
self.last_reported = 0
self.chunks_run_time = 0
self.total_chunks_processed = 0
self.report_interval = int_value(
conf.get('report_interval'), 3600)
self.max_chunks_per_second = int_value(
conf.get('chunks_per_second'), 30)
self.index_client = RdirClient(conf)
def init(self):
eventlet.monkey_patch(os=False)
self.session = requests.Session()
self.cs = ConscienceClient(self.conf)
self.rdir = RdirClient(self.conf)
self._acct_addr = None
self.acct_update = 0
self.graceful_timeout = 1
self.acct_refresh_interval = int_value(
self.conf.get('acct_refresh_interval'), 60
)
self.concurrency = int_value(self.conf.get('concurrency'), 1000)
self.acct_update = true_value(self.conf.get('acct_update', True))
self.rdir_update = true_value(self.conf.get('rdir_update', True))
super(EventWorker, self).init()
def run(self):
coros = []
queue_url = self.conf.get('queue_url', '127.0.0.1:11300')
concurrency = int_value(self.conf.get('concurrency'), 10)
server_gt = greenthread.getcurrent()
for i in range(concurrency):
beanstalk = Beanstalk.from_url(queue_url)
gt = eventlet.spawn(self.handle, beanstalk)
gt.link(_eventlet_stop, server_gt, beanstalk)
coros.append(gt)
beanstalk, gt = None, None
while self.alive:
self.notify()
try:
eventlet.sleep(1.0)
except AssertionError:
self.alive = False
break
self.notify()
try:
with Timeout(self.graceful_timeout) as t:
[c.kill(StopServe()) for c in coros]
[c.wait() for c in coros]
except Timeout as te:
if te != t:
raise
[c.kill() for c in coros]
def __init__(self, conf):
validate_service_conf(conf)
self.conf = conf
self.logger = get_logger(conf)
self.running = False
self.retry_interval = int_value(conf.get('retry_interval'), 30)
self.last_retry = 0
self.init_zmq()
self.init_queue()
self.init_workers()
def list_containers(self, account_id, limit=1000, marker=None,
end_marker=None, prefix=None, delimiter=None):
raw_list = self._raw_listing(account_id, limit=limit, marker=marker,
end_marker=end_marker, prefix=prefix,
delimiter=delimiter)
pipeline = self.conn.pipeline(True)
for container in raw_list:
pipeline.hmget(AccountBackend.ckey(account_id, container[0]),
'objects', 'bytes')
res = pipeline.execute()
i = 0
for container in raw_list:
if not container[3]:
container[1] = int_value(res[i][0], 0)
container[2] = int_value(res[i][1], 0)
i += 1
return raw_list
def __init__(self, conf_file, worker_class, **kwargs):
section_name = 'event-agent'
self.conf = read_conf(conf_file, section_name)
self.logger = get_logger(
self.conf, verbose=kwargs.pop('verbose', False))
redirect_stdio(self.logger)
drop_privileges(self.conf.get('user', 'openio'))
self.num_workers = int_value(self.conf.get('workers'), CPU_COUNT)
self.worker_class = worker_class
self.workers = {}
self.sig_queue = []
def __init__(self, conf, logger, volume):
self.conf = conf
self.logger = logger
self.volume = volume
self.run_time = 0
self.passes = 0
self.errors = 0
self.orphan_chunks = 0
self.faulty_chunks = 0
self.corrupted_chunks = 0
self.last_reported = 0
self.chunks_run_time = 0
self.bytes_running_time = 0
self.bytes_processed = 0
self.total_bytes_processed = 0
self.total_chunks_processed = 0
self.report_interval = int_value(conf.get("report_interval"), 3600)
self.max_chunks_per_second = int_value(conf.get("chunks_per_second"), 30)
self.max_bytes_per_second = int_value(conf.get("bytes_per_second"), 10000000)
self.container_client = ContainerClient(conf)
def __init__(self, conf, **kwargs):
super(BlobIndexer, self).__init__(conf)
self.logger = get_logger(conf)
volume = conf.get('volume')
if not volume:
raise exc.ConfigurationException('No volume specified for indexer')
self.volume = volume
self.passes = 0
self.errors = 0
self.last_reported = 0
self.chunks_run_time = 0
self.total_chunks_processed = 0
self.interval = int_value(
conf.get('interval'), 300)
self.report_interval = int_value(
conf.get('report_interval'), 3600)
self.max_chunks_per_second = int_value(
conf.get('chunks_per_second'), 30)
self.index_client = RdirClient(conf)
self.namespace, self.volume_id = check_volume(self.volume)
def __init__(self, conf, logger):
self.conf = conf
self.logger = logger
self.account = conf[CONF_ACCOUNT]
self.container_client = ContainerClient(self.conf)
self.account_client = AccountClient(self.conf)
self.content_factory = ContentFactory(self.conf)
self.passes = 0
self.errors = 0
self.last_reported = 0
self.contents_run_time = 0
self.total_contents_processed = 0
self.report_interval = int_value(
conf.get('report_interval'), 3600)
self.max_contents_per_second = int_value(
conf.get('contents_per_second'), 30)
self.container_fetch_limit = int_value(
conf.get('container_fetch_limit'), 100)
self.content_fetch_limit = int_value(
conf.get('content_fetch_limit'), 100)
self.outdated_threshold = int_value(
conf.get(CONF_OUTDATED_THRESHOLD), 9999999999)
self.new_policy = conf.get(CONF_NEW_POLICY)
def __init__(self, conf, name, context, **kwargs):
self.conf = conf
self.name = name
verbose = kwargs.pop("verbose", False)
self.logger = get_logger(self.conf, verbose=verbose)
self.init_zmq(context)
self.cs = ConscienceClient(self.conf)
self.rdir = RdirClient(self.conf)
self._acct_addr = None
self.acct_update = 0
self.acct_refresh_interval = int_value(conf.get("acct_refresh_interval"), 60)
self.acct_update = true_value(conf.get("acct_update", True))
self.rdir_update = true_value(conf.get("rdir_update", True))
self.session = requests.Session()
self.failed = False
def __init__(self, conf, name, context, **kwargs):
self.conf = conf
self.name = name
verbose = kwargs.pop('verbose', False)
self.logger = get_logger(self.conf, verbose=verbose)
self.init_zmq(context)
self.cs = ConscienceClient(self.conf)
self.rdir = RdirClient(self.conf)
self._acct_addr = None
self.acct_update = 0
self.acct_refresh_interval = int_value(
conf.get('acct_refresh_interval'), 60)
self.acct_update = true_value(conf.get('acct_update', True))
self.rdir_update = true_value(conf.get('rdir_update', True))
self.session = requests.Session()
self.failed = False
def init(self):
eventlet.monkey_patch(os=False)
self.tube = self.conf.get("tube", DEFAULT_TUBE)
self.session = requests.Session()
self.cs = ConscienceClient(self.conf)
self.rdir = RdirClient(self.conf)
self._acct_addr = None
self.acct_update = 0
self.graceful_timeout = 1
self.acct_refresh_interval = int_value(
self.conf.get('acct_refresh_interval'), 60
)
self.acct_update = true_value(self.conf.get('acct_update', True))
self.rdir_update = true_value(self.conf.get('rdir_update', True))
if 'handlers_conf' not in self.conf:
raise ValueError("'handlers_conf' path not defined in conf")
self.handlers = loadhandlers(
self.conf.get('handlers_conf'), evt_types, app=self)
super(EventWorker, self).init()
def init(self):
eventlet.monkey_patch(os=False)
self.tube = self.conf.get("tube", DEFAULT_TUBE)
self.session = requests.Session()
self.cs = ConscienceClient(self.conf)
self.rdir = RdirClient(self.conf)
self._acct_addr = None
self.acct_update = 0
self.graceful_timeout = 1
self.acct_refresh_interval = int_value(
self.conf.get('acct_refresh_interval'), 60)
self.acct_update = true_value(self.conf.get('acct_update', True))
self.rdir_update = true_value(self.conf.get('rdir_update', True))
if 'handlers_conf' not in self.conf:
raise ValueError("'handlers_conf' path not defined in conf")
self.handlers = loadhandlers(self.conf.get('handlers_conf'),
global_conf=self.conf,
app=self)
super(EventWorker, self).init()
def info_account(self, account_id):
conn = self.conn
if not account_id:
return None
account_id = conn.hget('account:%s' % account_id, 'id')
if not account_id:
return None
pipeline = conn.pipeline(False)
pipeline.hgetall('account:%s' % account_id)
pipeline.zcard('containers:%s' % account_id)
pipeline.hgetall('metadata:%s' % account_id)
data = pipeline.execute()
info = data[0]
for r in ['bytes', 'objects']:
info[r] = int_value(info[r], 0)
info['containers'] = data[1]
info['metadata'] = data[2]
return info
def info_account(self, account_id):
conn = self.conn
if not account_id:
return None
account_id = conn.hget('account:%s' % account_id, 'id')
if not account_id:
return None
pipeline = conn.pipeline(False)
pipeline.hgetall('account:%s' % account_id)
pipeline.zcard('containers:%s' % account_id)
pipeline.hgetall('metadata:%s' % account_id)
data = pipeline.execute()
info = data[0]
for r in ['bytes', 'objects']:
info[r] = int_value(info[r], 0)
info['containers'] = data[1]
info['metadata'] = data[2]
return info
def update_container(self, account_id, name, mtime, dtime, object_count,
bytes_used):
conn = self.conn
if not account_id or not name:
return None
_account_id = conn.hget('account:%s' % account_id, 'id')
if not _account_id:
if self.autocreate:
self.create_account(account_id)
else:
return None
lock = acquire_lock_with_timeout(conn,
AccountBackend.ckey(account_id, name),
1)
if not lock:
return None
data = conn.hgetall(AccountBackend.ckey(account_id, name))
record = {
'name': name,
'mtime': mtime or dtime,
'dtime': dtime,
'objects': object_count,
'bytes': bytes_used
}
if data:
data['mtime'] = Timestamp(data['mtime'])
data['dtime'] = Timestamp(data['dtime'])
for r in ['name', 'mtime', 'dtime', 'objects', 'bytes']:
if record[r] is None and data[r] is not None:
record[r] = data[r]
if data['mtime'] > record['mtime']:
record['mtime'] = data['mtime']
if data['dtime'] > record['dtime']:
record['dtime'] = data['dtime']
deleted = record['dtime'] >= record['mtime']
if not deleted:
incr_bytes_used = int_value(record.get('bytes'), 0) -\
int_value(data.get('bytes'), 0)
incr_object_count = int_value(record.get('objects'), 0) -\
int_value(data.get('objects'), 0)
elif record.get('mtime') > data.get('mtime'):
incr_bytes_used = -int_value(data.get('bytes'), 0)
incr_object_count = -int_value(data.get('objects'), 0)
else:
# The event has been delayed, the container has already
# been deleted, and the object and bytes statistics have
# already been reported to the account.
incr_bytes_used = 0
incr_object_count = 0
record.update({'name': name, 'account_uid': account_id})
# replace None values
for r in ['bytes', 'objects', 'mtime', 'dtime']:
if record[r] is None:
record[r] = 0
ct = {name: 0}
pipeline = conn.pipeline(True)
pipeline.hmset(AccountBackend.ckey(account_id, name), record)
if deleted:
pipeline.expire(AccountBackend.ckey(account_id, name), EXPIRE_TIME)
pipeline.zrem('containers:%s' % account_id, name)
else:
pipeline.persist(AccountBackend.ckey(account_id, name))
pipeline.zadd('containers:%s' % account_id, **ct)
if incr_object_count:
pipeline.hincrby('account:%s' % account_id, 'objects',
incr_object_count)
if incr_bytes_used:
pipeline.hincrby('account:%s' % account_id, 'bytes',
incr_bytes_used)
pipeline.execute()
release_lock(conn, AccountBackend.ckey(account_id, name), lock)
return name
def init_workers(self):
nbworkers = int_value(self.conf.get('workers'), 2)
workers = []
for i in xrange(nbworkers):
workers.append(EventWorker(self.conf, str(i), self.context))
self.workers = workers
def update_container(self, account_id, name, mtime, dtime, object_count,
bytes_used):
conn = self.conn
if not account_id or not name:
return None
_account_id = conn.hget('account:%s' % account_id, 'id')
if not _account_id:
if self.autocreate:
self.create_account(account_id)
else:
return None
lock = acquire_lock_with_timeout(
conn, 'container:%s:%s' % (account_id, name), 1)
if not lock:
return None
data = conn.hgetall('container:%s:%s' % (account_id, name))
record = {
'name': name,
'mtime': mtime,
'dtime': dtime,
'objects': object_count,
'bytes': bytes_used
}
deleted = False
if data:
data['mtime'] = Timestamp(data['mtime'])
data['dtime'] = Timestamp(data['dtime'])
for r in ['name', 'mtime', 'dtime', 'objects', 'bytes']:
if record[r] is None and data[r] is not None:
record[r] = data[r]
if data['mtime'] > record['mtime']:
record['mtime'] = data['mtime']
if data['dtime'] > record['dtime']:
record['dtime'] = data['dtime']
if record['dtime'] > record['mtime']:
deleted = True
if not deleted:
incr_bytes_used = int_value(record.get('bytes'), 0) -\
int_value(data.get('bytes'), 0)
incr_object_count = int_value(record.get('objects'), 0) -\
int_value(data.get('objects'), 0)
else:
incr_bytes_used = -int_value(data.get('bytes'), 0)
incr_object_count = -int_value(data.get('objects'), 0)
record.update({'name': name, 'account_uid': account_id})
# replace None values
for r in ['bytes', 'objects', 'mtime', 'dtime']:
if record[r] is None:
record[r] = 0
ct = {str(name): 0}
pipeline = conn.pipeline(True)
if deleted:
pipeline.delete('container:%s:%s' % (account_id, name))
pipeline.zrem('containers:%s' % account_id, str(name))
else:
pipeline.hmset('container:%s:%s' % (account_id, name), record)
pipeline.zadd('containers:%s' % account_id, **ct)
if incr_object_count:
pipeline.hincrby('account:%s' % account_id, 'objects',
incr_object_count)
if incr_bytes_used:
pipeline.hincrby('account:%s' % account_id, 'bytes',
incr_bytes_used)
pipeline.execute()
release_lock(conn, 'container:%s:%s' % (account_id, name), lock)
return name
def update_container(self, account_id, name, mtime, dtime, object_count,
bytes_used):
conn = self.conn
if not account_id or not name:
return None
_account_id = conn.hget('account:%s' % account_id, 'id')
if not _account_id:
if self.autocreate:
self.create_account(account_id)
else:
return None
lock = acquire_lock_with_timeout(
conn, AccountBackend.ckey(account_id, name), 1)
if not lock:
return None
data = conn.hgetall(AccountBackend.ckey(account_id, name))
record = {'name': name, 'mtime': mtime, 'dtime': dtime,
'objects': object_count, 'bytes': bytes_used}
deleted = False
if data:
data['mtime'] = Timestamp(data['mtime'])
data['dtime'] = Timestamp(data['dtime'])
for r in ['name', 'mtime', 'dtime', 'objects', 'bytes']:
if record[r] is None and data[r] is not None:
record[r] = data[r]
if data['mtime'] > record['mtime']:
record['mtime'] = data['mtime']
if data['dtime'] > record['dtime']:
record['dtime'] = data['dtime']
if record['dtime'] > record['mtime']:
deleted = True
if not deleted:
incr_bytes_used = int_value(record.get('bytes'), 0) -\
int_value(data.get('bytes'), 0)
incr_object_count = int_value(record.get('objects'), 0) -\
int_value(data.get('objects'), 0)
else:
incr_bytes_used = - int_value(data.get('bytes'), 0)
incr_object_count = - int_value(data.get('objects'), 0)
record.update({
'name': name,
'account_uid': account_id
})
# replace None values
for r in ['bytes', 'objects', 'mtime', 'dtime']:
if record[r] is None:
record[r] = 0
ct = {name: 0}
pipeline = conn.pipeline(True)
pipeline.hmset(AccountBackend.ckey(account_id, name), record)
if deleted:
pipeline.expire(AccountBackend.ckey(account_id, name), EXPIRE_TIME)
pipeline.zrem('containers:%s' % account_id, name)
else:
pipeline.persist(AccountBackend.ckey(account_id, name))
pipeline.zadd('containers:%s' % account_id, **ct)
if incr_object_count:
pipeline.hincrby('account:%s' % account_id, 'objects',
incr_object_count)
if incr_bytes_used:
pipeline.hincrby('account:%s' % account_id, 'bytes',
incr_bytes_used)
pipeline.execute()
release_lock(conn, AccountBackend.ckey(account_id, name), lock)
return name
def init_workers(self):
nbworkers = int_value(self.conf.get('workers'), 2)
workers = []
for i in xrange(nbworkers):
workers.append(EventWorker(self.conf, str(i), self.context))
self.workers = workers
