def __init__(self, conf, **kwargs):
super(StorageTierer, self).__init__(conf)
self.logger = get_logger(conf)
if not conf.get(CONF_ACCOUNT):
raise exc.ConfigurationException(
"No account specified for storage tiering "
"(token '%s'" % CONF_ACCOUNT)
if not conf.get(CONF_OUTDATED_THRESHOLD):
raise exc.ConfigurationException(
"No date specified for storage tiering "
"(token '%s'" % CONF_OUTDATED_THRESHOLD)
if not conf.get(CONF_NEW_POLICY):
raise exc.ConfigurationException(
"No new policy specified for storage tiering "
"(token '%s'" % CONF_NEW_POLICY)
def __init__(self, conf, **kwargs):
super(BlobIndexer, self).__init__(conf)
self.logger = get_logger(conf)
volume = conf.get('volume')
if not volume:
raise exc.ConfigurationException('No volume specified for indexer')
self.volume = volume
self.passes = 0
self.errors = 0
self.successes = 0
self.last_reported = 0
self.total_since_last_reported = 0
self.chunks_run_time = 0
self.interval = int_value(
conf.get('interval'), 300)
self.report_interval = int_value(
conf.get('report_interval'), 3600)
self.max_chunks_per_second = int_value(
conf.get('chunks_per_second'), 30)
pm = get_pool_manager(pool_connections=10)
self.index_client = RdirClient(conf, logger=self.logger,
pool_manager=pm)
self.namespace, self.volume_id = check_volume(self.volume)
self.convert_chunks = true_value(conf.get('convert_chunks'))
if self.convert_chunks:
converter_conf = self.conf.copy()
converter_conf['no_backup'] = True
self.converter = BlobConverter(converter_conf, logger=self.logger,
pool_manager=pm)
else:
self.converter = None
def __init__(self, conf, **kwargs):
super(BlobAuditor, self).__init__(conf)
self.logger = get_logger(conf)
volume = conf.get('volume')
if not volume:
raise exc.ConfigurationException('No volume specified for auditor')
self.volume = volume
def __init__(self, conf, **kwargs):
super(StorageTierer, self).__init__(conf)
self.logger = get_logger(conf)
if not conf.get(CONF_ACCOUNT):
raise exc.ConfigurationException(
"No account specified for storage tiering "
"(token '%s'" % CONF_ACCOUNT)
if not conf.get(CONF_OUTDATED_THRESHOLD):
raise exc.ConfigurationException(
"No threshold specified for storage tiering "
"(token '%s'" % CONF_OUTDATED_THRESHOLD)
if not conf.get(CONF_NEW_POLICY):
raise exc.ConfigurationException(
"No new policy specified for storage tiering "
"(token '%s'" % CONF_NEW_POLICY)
if conf.get('syslog_prefix'):
print("Logging to syslog, with prefix '%(syslog_prefix)s'" % conf)
def get_credentials(storage_method, application_key_path=None,
renew=False):
if not application_key_path:
application_key_path = '/etc/oio/sds/b2-appkey.conf'
if not storage_method.bucket_name:
message = "missing backblaze parameters: %s" % ('bucket_name',)
raise oioexc.ConfigurationException(message)
if not storage_method.account_id:
message = "missing backblaze parameters: %s" % ('account_id',)
raise oioexc.ConfigurationException(message)
if not application_key_path:
message = "missing backblaze parameters: %s" % \
('application_key_path',)
raise oioexc.ConfigurationException(message)
key = '%s.%s' % (storage_method.account_id, storage_method.bucket_name)
if not renew:
authorization = BackblazeUtils.b2_authorization_list.get(key, None)
if authorization:
return authorization
config = ConfigParser.ConfigParser()
app_key = None
with open(application_key_path) as app_key_f:
try:
config.readfp(app_key_f)
except IOError as exc:
raise oioexc.ConfigurationException(
"Failed to load application key: %s"
% exc)
app_key = config.get('backblaze',
'%s.%s.application_key'
% (storage_method.account_id,
storage_method.bucket_name))
if not app_key:
raise oioexc.ConfigurationException('application key not found')
meta = {}
meta['backblaze.account_id'] = storage_method.account_id
meta['backblaze.application_key'] = app_key
meta['bucket_name'] = storage_method.bucket_name
backblaze = Backblaze(storage_method.account_id,
app_key)
meta['authorization'] = backblaze.authorization_token
meta['upload_token'] = backblaze._get_upload_token_by_bucket_name(
storage_method.bucket_name)
BackblazeUtils.b2_authorization_list[key] = meta
return meta
def configure(self):
for k in ['host', 'port']:
if k not in self.checker_conf:
raise exc.ConfigurationException(
'Missing field "%s" in configuration' % k)
addrinfo = socket.getaddrinfo(
self.checker_conf['host'], self.checker_conf['port'],
socktype=socket.SOCK_STREAM, flags=socket.AI_NUMERICHOST)[0]
self.family, _, _, _, self.addr = addrinfo
def __init__(self, conf, logger):
super(TcpChecker, self).__init__(conf, logger)
for k in ['host', 'port']:
if k not in conf:
raise exc.ConfigurationException(
'Missing field "%s" in configuration' % k)
self.conf = conf
self.addr = (self.conf['host'], self.conf['port'])
def __init__(self, conf, **kwargs):
super(BlobMover, self).__init__(conf)
self.logger = get_logger(conf)
volume = conf.get('volume')
if not volume:
raise exc.ConfigurationException('No volume specified for mover')
self.volume = volume
global SLEEP_TIME
if SLEEP_TIME > int(conf.get('report_interval', 3600)):
SLEEP_TIME = int(conf.get('report_interval', 3600))
def _configure(self):
for k in ('uri', ):
if k not in self.checker_conf:
raise exc.ConfigurationException(
'Missing field "%s" in configuration' % k)
self.path = self.checker_conf['uri'].lstrip('/')
self.name = '%s|%s' % (self.name, self.path)
self.url = '%s:%s%s%s' % (self.host, self.port,
'' if self.path.startswith('/') else '/',
self.path)
def configure(self):
for k in ['host', 'port', 'uri']:
if k not in self.checker_conf:
raise exc.ConfigurationException(
'Missing field "%s" in configuration' % k)
self.host = self.checker_conf['host']
self.port = self.checker_conf['port']
self.path = self.checker_conf['uri']
self.name = '%s|http|%s|%s|%s' % \
(self.srv_type, self.host, self.port, self.path)
self.url = '%s:%s/%s' % (self.host, self.port, self.path)
def configure(self):
for k in ['host', 'port', 'uri']:
if k not in self.checker_conf:
raise exc.ConfigurationException(
'Missing field "%s" in configuration' % k)
self.host = self.checker_conf['host']
self.port = self.checker_conf['port']
self.uri = self.checker_conf['uri']
self.name = 'http|%s|%s|%s' % (self.host, self.port, self.uri)
self.url = 'http://%s:%s/%s' % \
(self.host, self.port, self.uri.lstrip('/'))
self.session = requests.session()
def __init__(self, conf, logger):
super(HttpChecker, self).__init__(conf, logger)
for k in ['host', 'port', 'uri']:
if k not in conf:
raise exc.ConfigurationException(
'Missing field "%s" in configuration' % k)
self.host = conf['host']
self.port = conf['port']
self.uri = conf['uri']
self.name = 'http|%s|%s|%s' % (self.host, self.port, self.uri)
self.url = 'http://%s:%s/%s' % \
(self.host, self.port, self.uri.lstrip('/'))
def __init__(self, conf, **kwargs):
super(BlobIndexer, self).__init__(conf)
self.logger = get_logger(conf)
volume = conf.get('volume')
if not volume:
raise exc.ConfigurationException('No volume specified for indexer')
self.volume = volume
self.passes = 0
self.errors = 0
self.successes = 0
self.last_reported = 0
self.chunks_run_time = 0
self.interval = int_value(conf.get('interval'), 300)
self.report_interval = int_value(conf.get('report_interval'), 3600)
self.max_chunks_per_second = int_value(conf.get('chunks_per_second'),
30)
self.index_client = RdirClient(conf, logger=self.logger)
self.namespace, self.volume_id = check_volume(self.volume)
def get_flatns_manager(self):
if self._flatns_manager:
return self._flatns_manager
self.info()
options = self._nsinfo['options']
bitlength, offset, size = None, 0, 0
try:
bitlength = int(options['flat_bitlength'])
except:
raise exceptions.ConfigurationException(
"Namespace not configured for autocontainers")
try:
if 'flat_hash_offset' in options:
offset = int(options['flat_hash_offset'])
if 'flat_hash_size' in options:
size = int(options['flat_hash_size'])
except:
raise Exception("Invalid autocontainer config: offset/size")
self._flatns_manager = HashedContainerBuilder(offset=offset,
size=size,
bits=bitlength)
return self._flatns_manager
def __init__(self, agent, checker_conf, logger):
self.agent = agent
self.checker_conf = checker_conf
self.logger = logger
self.timeout = float_value(checker_conf.get('timeout'), 5.0)
self.rise = checker_conf['rise']
self.fall = checker_conf['fall']
self.results = RingBuffer(max([self.rise, self.fall]))
self.name = checker_conf.get('name')
self.srv_type = agent.service['type']
self.last_result = None
for k in ('host', 'port'):
if k not in self.checker_conf:
raise exc.ConfigurationException(
'Missing field "%s" in configuration' % k)
self.host = self.checker_conf['host']
self.port = self.checker_conf['port']
self.name = '%s|%s|%s|%s' % \
(self.srv_type, self.checker_type, self.host, self.port)
self.last_check_success = True
self._configure()
def _b2_credentials(self, storage_method, key_file):
key_file = key_file or '/etc/oio/sds/b2-appkey.conf'
try:
return BackblazeUtils.get_credentials(storage_method, key_file)
except BackblazeUtilsException as err:
raise exc.ConfigurationException(str(err))
def configure(self):
for k in ['host', 'port']:
if k not in self.checker_conf:
raise exc.ConfigurationException(
'Missing field "%s" in configuration' % k)
self.addr = (self.checker_conf['host'], self.checker_conf['port'])
def configure(self):
super(BeanstalkdStat, self).configure()
for k in ('host', 'port'):
if k not in self.stat_conf:
raise exc.ConfigurationException(
'Missing field "%s" in configuration' % k)
