def __init__(self, conf, logger, volume): self.conf = conf self.logger = logger or get_logger(conf) self.volume = volume self.run_time = 0 self.passes = 0 self.errors = 0 self.last_reported = 0 self.last_usage_check = 0 self.chunks_run_time = 0 self.bytes_running_time = 0 self.bytes_processed = 0 self.total_bytes_processed = 0 self.total_chunks_processed = 0 self.usage_target = int_value( conf.get('usage_target'), 0) self.usage_check_interval = int_value( conf.get('usage_check_interval'), 3600) self.report_interval = int_value( conf.get('report_interval'), 3600) self.max_chunks_per_second = int_value( conf.get('chunks_per_second'), 30) self.max_bytes_per_second = int_value( conf.get('bytes_per_second'), 10000000) self.blob_client = BlobClient() self.container_client = ContainerClient(conf)
def __init__(self, conf, logger, volume): self.conf = conf self.logger = logger or get_logger(conf) self.volume = volume self.run_time = 0 self.passes = 0 self.errors = 0 self.last_reported = 0 self.chunks_run_time = 0 self.bytes_running_time = 0 self.bytes_processed = 0 self.total_bytes_processed = 0 self.total_chunks_processed = 0 self.dry_run = true_value( conf.get('dry_run', False)) self.report_interval = int_value( conf.get('report_interval'), 3600) self.max_chunks_per_second = int_value( conf.get('chunks_per_second'), 30) self.max_bytes_per_second = int_value( conf.get('bytes_per_second'), 10000000) self.rdir_fetch_limit = int_value( conf.get('rdir_fetch_limit'), 100) self.rdir_client = RdirClient(conf) self.content_factory = ContentFactory(conf)
def __init__(self, conf, service, **kwargs): self.conf = conf self.running = False for k in ['host', 'port', 'type']: if k not in service: raise Exception( 'Missing field "%s" in service configuration' % k) self.name = '%s|%s' % \ (service['host'], service['port']) self.check_interval = float_value(conf.get('check_interval'), 1) self.service = service self.rise = int_value(conf.get('rise'), 1) self.fall = int_value(conf.get('fall'), 1) self.logger = get_logger(self.conf) self.cs = ConscienceClient(self.conf) self.client = Client(self.conf) self.last_status = False self.failed = False self.service_definition = { 'ns': self.conf['namespace'], 'type': self.service['type'], 'addr': '%s:%s' % (self.service['host'], self.service['port']), 'score': 0, 'tags': {}} if self.service.get('location', None): self.service_definition['tags']['tag.loc'] = \ self.service['location'] self.service_checks = list() self.service_stats = list() self.init_checkers(service) self.init_stats(service)
def __init__(self, conf): validate_service_conf(conf) self.conf = conf self.logger = get_logger(conf) self.running = False self.retries_run_time = 0 self.max_retries_per_second = int_value( conf.get('retries_per_second'), 30) self.batch_size = int_value(conf.get('batch_size'), 500) self.init_zmq() self.init_queue() self.init_workers()
def __init__(self, conf, logger, volume): self.conf = conf self.logger = logger self.volume = volume self.passes = 0 self.errors = 0 self.last_reported = 0 self.chunks_run_time = 0 self.total_chunks_processed = 0 self.report_interval = int_value( conf.get('report_interval'), 3600) self.max_chunks_per_second = int_value( conf.get('chunks_per_second'), 30) self.index_client = RdirClient(conf)
def init(self): eventlet.monkey_patch(os=False) self.session = requests.Session() self.cs = ConscienceClient(self.conf) self.rdir = RdirClient(self.conf) self._acct_addr = None self.acct_update = 0 self.graceful_timeout = 1 self.acct_refresh_interval = int_value( self.conf.get('acct_refresh_interval'), 60 ) self.concurrency = int_value(self.conf.get('concurrency'), 1000) self.acct_update = true_value(self.conf.get('acct_update', True)) self.rdir_update = true_value(self.conf.get('rdir_update', True)) super(EventWorker, self).init()
def run(self): coros = [] queue_url = self.conf.get('queue_url', '127.0.0.1:11300') concurrency = int_value(self.conf.get('concurrency'), 10) server_gt = greenthread.getcurrent() for i in range(concurrency): beanstalk = Beanstalk.from_url(queue_url) gt = eventlet.spawn(self.handle, beanstalk) gt.link(_eventlet_stop, server_gt, beanstalk) coros.append(gt) beanstalk, gt = None, None while self.alive: self.notify() try: eventlet.sleep(1.0) except AssertionError: self.alive = False break self.notify() try: with Timeout(self.graceful_timeout) as t: [c.kill(StopServe()) for c in coros] [c.wait() for c in coros] except Timeout as te: if te != t: raise [c.kill() for c in coros]
def __init__(self, conf): validate_service_conf(conf) self.conf = conf self.logger = get_logger(conf) self.running = False self.retry_interval = int_value(conf.get('retry_interval'), 30) self.last_retry = 0 self.init_zmq() self.init_queue() self.init_workers()
def list_containers(self, account_id, limit=1000, marker=None, end_marker=None, prefix=None, delimiter=None): raw_list = self._raw_listing(account_id, limit=limit, marker=marker, end_marker=end_marker, prefix=prefix, delimiter=delimiter) pipeline = self.conn.pipeline(True) for container in raw_list: pipeline.hmget(AccountBackend.ckey(account_id, container[0]), 'objects', 'bytes') res = pipeline.execute() i = 0 for container in raw_list: if not container[3]: container[1] = int_value(res[i][0], 0) container[2] = int_value(res[i][1], 0) i += 1 return raw_list
def __init__(self, conf_file, worker_class, **kwargs): section_name = 'event-agent' self.conf = read_conf(conf_file, section_name) self.logger = get_logger( self.conf, verbose=kwargs.pop('verbose', False)) redirect_stdio(self.logger) drop_privileges(self.conf.get('user', 'openio')) self.num_workers = int_value(self.conf.get('workers'), CPU_COUNT) self.worker_class = worker_class self.workers = {} self.sig_queue = []
def __init__(self, conf, logger, volume): self.conf = conf self.logger = logger self.volume = volume self.run_time = 0 self.passes = 0 self.errors = 0 self.orphan_chunks = 0 self.faulty_chunks = 0 self.corrupted_chunks = 0 self.last_reported = 0 self.chunks_run_time = 0 self.bytes_running_time = 0 self.bytes_processed = 0 self.total_bytes_processed = 0 self.total_chunks_processed = 0 self.report_interval = int_value(conf.get("report_interval"), 3600) self.max_chunks_per_second = int_value(conf.get("chunks_per_second"), 30) self.max_bytes_per_second = int_value(conf.get("bytes_per_second"), 10000000) self.container_client = ContainerClient(conf)
def __init__(self, conf, **kwargs): super(BlobIndexer, self).__init__(conf) self.logger = get_logger(conf) volume = conf.get('volume') if not volume: raise exc.ConfigurationException('No volume specified for indexer') self.volume = volume self.passes = 0 self.errors = 0 self.last_reported = 0 self.chunks_run_time = 0 self.total_chunks_processed = 0 self.interval = int_value( conf.get('interval'), 300) self.report_interval = int_value( conf.get('report_interval'), 3600) self.max_chunks_per_second = int_value( conf.get('chunks_per_second'), 30) self.index_client = RdirClient(conf) self.namespace, self.volume_id = check_volume(self.volume)
def __init__(self, conf, logger): self.conf = conf self.logger = logger self.account = conf[CONF_ACCOUNT] self.container_client = ContainerClient(self.conf) self.account_client = AccountClient(self.conf) self.content_factory = ContentFactory(self.conf) self.passes = 0 self.errors = 0 self.last_reported = 0 self.contents_run_time = 0 self.total_contents_processed = 0 self.report_interval = int_value( conf.get('report_interval'), 3600) self.max_contents_per_second = int_value( conf.get('contents_per_second'), 30) self.container_fetch_limit = int_value( conf.get('container_fetch_limit'), 100) self.content_fetch_limit = int_value( conf.get('content_fetch_limit'), 100) self.outdated_threshold = int_value( conf.get(CONF_OUTDATED_THRESHOLD), 9999999999) self.new_policy = conf.get(CONF_NEW_POLICY)
def __init__(self, conf, name, context, **kwargs): self.conf = conf self.name = name verbose = kwargs.pop("verbose", False) self.logger = get_logger(self.conf, verbose=verbose) self.init_zmq(context) self.cs = ConscienceClient(self.conf) self.rdir = RdirClient(self.conf) self._acct_addr = None self.acct_update = 0 self.acct_refresh_interval = int_value(conf.get("acct_refresh_interval"), 60) self.acct_update = true_value(conf.get("acct_update", True)) self.rdir_update = true_value(conf.get("rdir_update", True)) self.session = requests.Session() self.failed = False
def __init__(self, conf, name, context, **kwargs): self.conf = conf self.name = name verbose = kwargs.pop('verbose', False) self.logger = get_logger(self.conf, verbose=verbose) self.init_zmq(context) self.cs = ConscienceClient(self.conf) self.rdir = RdirClient(self.conf) self._acct_addr = None self.acct_update = 0 self.acct_refresh_interval = int_value( conf.get('acct_refresh_interval'), 60) self.acct_update = true_value(conf.get('acct_update', True)) self.rdir_update = true_value(conf.get('rdir_update', True)) self.session = requests.Session() self.failed = False
def init(self): eventlet.monkey_patch(os=False) self.tube = self.conf.get("tube", DEFAULT_TUBE) self.session = requests.Session() self.cs = ConscienceClient(self.conf) self.rdir = RdirClient(self.conf) self._acct_addr = None self.acct_update = 0 self.graceful_timeout = 1 self.acct_refresh_interval = int_value( self.conf.get('acct_refresh_interval'), 60 ) self.acct_update = true_value(self.conf.get('acct_update', True)) self.rdir_update = true_value(self.conf.get('rdir_update', True)) if 'handlers_conf' not in self.conf: raise ValueError("'handlers_conf' path not defined in conf") self.handlers = loadhandlers( self.conf.get('handlers_conf'), evt_types, app=self) super(EventWorker, self).init()
def init(self): eventlet.monkey_patch(os=False) self.tube = self.conf.get("tube", DEFAULT_TUBE) self.session = requests.Session() self.cs = ConscienceClient(self.conf) self.rdir = RdirClient(self.conf) self._acct_addr = None self.acct_update = 0 self.graceful_timeout = 1 self.acct_refresh_interval = int_value( self.conf.get('acct_refresh_interval'), 60) self.acct_update = true_value(self.conf.get('acct_update', True)) self.rdir_update = true_value(self.conf.get('rdir_update', True)) if 'handlers_conf' not in self.conf: raise ValueError("'handlers_conf' path not defined in conf") self.handlers = loadhandlers(self.conf.get('handlers_conf'), global_conf=self.conf, app=self) super(EventWorker, self).init()
def info_account(self, account_id): conn = self.conn if not account_id: return None account_id = conn.hget('account:%s' % account_id, 'id') if not account_id: return None pipeline = conn.pipeline(False) pipeline.hgetall('account:%s' % account_id) pipeline.zcard('containers:%s' % account_id) pipeline.hgetall('metadata:%s' % account_id) data = pipeline.execute() info = data[0] for r in ['bytes', 'objects']: info[r] = int_value(info[r], 0) info['containers'] = data[1] info['metadata'] = data[2] return info
def update_container(self, account_id, name, mtime, dtime, object_count, bytes_used): conn = self.conn if not account_id or not name: return None _account_id = conn.hget('account:%s' % account_id, 'id') if not _account_id: if self.autocreate: self.create_account(account_id) else: return None lock = acquire_lock_with_timeout(conn, AccountBackend.ckey(account_id, name), 1) if not lock: return None data = conn.hgetall(AccountBackend.ckey(account_id, name)) record = { 'name': name, 'mtime': mtime or dtime, 'dtime': dtime, 'objects': object_count, 'bytes': bytes_used } if data: data['mtime'] = Timestamp(data['mtime']) data['dtime'] = Timestamp(data['dtime']) for r in ['name', 'mtime', 'dtime', 'objects', 'bytes']: if record[r] is None and data[r] is not None: record[r] = data[r] if data['mtime'] > record['mtime']: record['mtime'] = data['mtime'] if data['dtime'] > record['dtime']: record['dtime'] = data['dtime'] deleted = record['dtime'] >= record['mtime'] if not deleted: incr_bytes_used = int_value(record.get('bytes'), 0) -\ int_value(data.get('bytes'), 0) incr_object_count = int_value(record.get('objects'), 0) -\ int_value(data.get('objects'), 0) elif record.get('mtime') > data.get('mtime'): incr_bytes_used = -int_value(data.get('bytes'), 0) incr_object_count = -int_value(data.get('objects'), 0) else: # The event has been delayed, the container has already # been deleted, and the object and bytes statistics have # already been reported to the account. incr_bytes_used = 0 incr_object_count = 0 record.update({'name': name, 'account_uid': account_id}) # replace None values for r in ['bytes', 'objects', 'mtime', 'dtime']: if record[r] is None: record[r] = 0 ct = {name: 0} pipeline = conn.pipeline(True) pipeline.hmset(AccountBackend.ckey(account_id, name), record) if deleted: pipeline.expire(AccountBackend.ckey(account_id, name), EXPIRE_TIME) pipeline.zrem('containers:%s' % account_id, name) else: pipeline.persist(AccountBackend.ckey(account_id, name)) pipeline.zadd('containers:%s' % account_id, **ct) if incr_object_count: pipeline.hincrby('account:%s' % account_id, 'objects', incr_object_count) if incr_bytes_used: pipeline.hincrby('account:%s' % account_id, 'bytes', incr_bytes_used) pipeline.execute() release_lock(conn, AccountBackend.ckey(account_id, name), lock) return name
def init_workers(self): nbworkers = int_value(self.conf.get('workers'), 2) workers = [] for i in xrange(nbworkers): workers.append(EventWorker(self.conf, str(i), self.context)) self.workers = workers
def update_container(self, account_id, name, mtime, dtime, object_count, bytes_used): conn = self.conn if not account_id or not name: return None _account_id = conn.hget('account:%s' % account_id, 'id') if not _account_id: if self.autocreate: self.create_account(account_id) else: return None lock = acquire_lock_with_timeout( conn, 'container:%s:%s' % (account_id, name), 1) if not lock: return None data = conn.hgetall('container:%s:%s' % (account_id, name)) record = { 'name': name, 'mtime': mtime, 'dtime': dtime, 'objects': object_count, 'bytes': bytes_used } deleted = False if data: data['mtime'] = Timestamp(data['mtime']) data['dtime'] = Timestamp(data['dtime']) for r in ['name', 'mtime', 'dtime', 'objects', 'bytes']: if record[r] is None and data[r] is not None: record[r] = data[r] if data['mtime'] > record['mtime']: record['mtime'] = data['mtime'] if data['dtime'] > record['dtime']: record['dtime'] = data['dtime'] if record['dtime'] > record['mtime']: deleted = True if not deleted: incr_bytes_used = int_value(record.get('bytes'), 0) -\ int_value(data.get('bytes'), 0) incr_object_count = int_value(record.get('objects'), 0) -\ int_value(data.get('objects'), 0) else: incr_bytes_used = -int_value(data.get('bytes'), 0) incr_object_count = -int_value(data.get('objects'), 0) record.update({'name': name, 'account_uid': account_id}) # replace None values for r in ['bytes', 'objects', 'mtime', 'dtime']: if record[r] is None: record[r] = 0 ct = {str(name): 0} pipeline = conn.pipeline(True) if deleted: pipeline.delete('container:%s:%s' % (account_id, name)) pipeline.zrem('containers:%s' % account_id, str(name)) else: pipeline.hmset('container:%s:%s' % (account_id, name), record) pipeline.zadd('containers:%s' % account_id, **ct) if incr_object_count: pipeline.hincrby('account:%s' % account_id, 'objects', incr_object_count) if incr_bytes_used: pipeline.hincrby('account:%s' % account_id, 'bytes', incr_bytes_used) pipeline.execute() release_lock(conn, 'container:%s:%s' % (account_id, name), lock) return name
def update_container(self, account_id, name, mtime, dtime, object_count, bytes_used): conn = self.conn if not account_id or not name: return None _account_id = conn.hget('account:%s' % account_id, 'id') if not _account_id: if self.autocreate: self.create_account(account_id) else: return None lock = acquire_lock_with_timeout( conn, AccountBackend.ckey(account_id, name), 1) if not lock: return None data = conn.hgetall(AccountBackend.ckey(account_id, name)) record = {'name': name, 'mtime': mtime, 'dtime': dtime, 'objects': object_count, 'bytes': bytes_used} deleted = False if data: data['mtime'] = Timestamp(data['mtime']) data['dtime'] = Timestamp(data['dtime']) for r in ['name', 'mtime', 'dtime', 'objects', 'bytes']: if record[r] is None and data[r] is not None: record[r] = data[r] if data['mtime'] > record['mtime']: record['mtime'] = data['mtime'] if data['dtime'] > record['dtime']: record['dtime'] = data['dtime'] if record['dtime'] > record['mtime']: deleted = True if not deleted: incr_bytes_used = int_value(record.get('bytes'), 0) -\ int_value(data.get('bytes'), 0) incr_object_count = int_value(record.get('objects'), 0) -\ int_value(data.get('objects'), 0) else: incr_bytes_used = - int_value(data.get('bytes'), 0) incr_object_count = - int_value(data.get('objects'), 0) record.update({ 'name': name, 'account_uid': account_id }) # replace None values for r in ['bytes', 'objects', 'mtime', 'dtime']: if record[r] is None: record[r] = 0 ct = {name: 0} pipeline = conn.pipeline(True) pipeline.hmset(AccountBackend.ckey(account_id, name), record) if deleted: pipeline.expire(AccountBackend.ckey(account_id, name), EXPIRE_TIME) pipeline.zrem('containers:%s' % account_id, name) else: pipeline.persist(AccountBackend.ckey(account_id, name)) pipeline.zadd('containers:%s' % account_id, **ct) if incr_object_count: pipeline.hincrby('account:%s' % account_id, 'objects', incr_object_count) if incr_bytes_used: pipeline.hincrby('account:%s' % account_id, 'bytes', incr_bytes_used) pipeline.execute() release_lock(conn, AccountBackend.ckey(account_id, name), lock) return name