def __init__(self, conf, logger, volume):
self.conf = conf
self.logger = logger or get_logger(conf)
self.volume = volume
self.run_time = 0
self.passes = 0
self.errors = 0
self.last_reported = 0
self.last_usage_check = 0
self.chunks_run_time = 0
self.bytes_running_time = 0
self.bytes_processed = 0
self.total_bytes_processed = 0
self.total_chunks_processed = 0
self.usage_target = int_value(
conf.get('usage_target'), 0)
self.usage_check_interval = int_value(
conf.get('usage_check_interval'), 3600)
self.report_interval = int_value(
conf.get('report_interval'), 3600)
self.max_chunks_per_second = int_value(
conf.get('chunks_per_second'), 30)
self.max_bytes_per_second = int_value(
conf.get('bytes_per_second'), 10000000)
self.blob_client = BlobClient()
self.container_client = ContainerClient(conf)
self.content_factory = ContentFactory(conf)
def __init__(self, conf, container_id, metadata, chunks, storage_method,
account, container_name, blob_client=None,
container_client=None, logger=None):
self.conf = conf
self.container_id = container_id
self.metadata = metadata
self.chunks = ChunksHelper(chunks)
self.storage_method = storage_method
self.logger = logger or get_logger(self.conf)
self.blob_client = (blob_client or BlobClient(conf))
self.container_client = (container_client
or ContainerClient(self.conf,
logger=self.logger))
# FIXME: all these may be properties
self.content_id = self.metadata["id"]
self.path = self.metadata["name"]
self.length = int(self.metadata["length"])
self.version = self.metadata["version"]
self.checksum = self.metadata["hash"]
self.chunk_method = self.metadata["chunk_method"]
self.account = account
self.container_name = container_name
if 'full_path' in self.metadata:
self.full_path = metadata['full_path']
else:
self.full_path = encode_fullpath(
self.account, self.container_name, self.path, self.version,
self.content_id)
def __init__(self, conf, logger, volume):
self.conf = conf
self.logger = logger or get_logger(conf)
self.volume = volume
self.namespace, self.address = check_volume(self.volume)
self.running = False
self.run_time = 0
self.passes = 0
self.errors = 0
self.last_reported = 0
self.last_usage_check = 0
self.chunks_run_time = 0
self.bytes_running_time = 0
self.bytes_processed = 0
self.total_bytes_processed = 0
self.total_chunks_processed = 0
self.concurrency = int_value(conf.get('concurrency'), 10)
self.usage_target = int_value(conf.get('usage_target'), 0)
self.usage_check_interval = int_value(conf.get('usage_check_interval'),
60)
self.report_interval = int_value(conf.get('report_interval'), 3600)
self.max_chunks_per_second = int_value(conf.get('chunks_per_second'),
30)
self.limit = int_value(conf.get('limit'), 0)
self.allow_links = true_value(conf.get('allow_links', True))
self.blob_client = BlobClient(conf)
self.container_client = ContainerClient(conf, logger=self.logger)
self.content_factory = ContentFactory(
conf,
container_client=self.container_client,
blob_client=self.blob_client)
self.excluded_rawx = \
[rawx for rawx in conf.get('excluded_rawx', '').split(',') if rawx]
self.fake_excluded_chunks = self._generate_fake_excluded_chunks()
def setUp(self):
super(TestBlobMover, self).setUp()
self.container = random_str(16)
self.cid = cid_from_name(self.account, self.container)
self.path = random_str(16)
self.api = ObjectStorageApi(self.ns)
self.blob_client = BlobClient(self.conf)
self.api.container_create(self.account, self.container)
_, chunks = self.api.container.content_prepare(self.account,
self.container,
self.path,
size=1)
services = self.conscience.all_services('rawx')
if len(chunks) >= len([s for s in services if s['score'] > 0]):
self.skipTest("need at least %d rawx to run" % (len(chunks) + 1))
self.rawx_volumes = dict()
for rawx in services:
tags = rawx['tags']
service_id = tags.get('tag.service_id', None)
if service_id is None:
service_id = rawx['addr']
volume = tags.get('tag.vol', None)
self.rawx_volumes[service_id] = volume
self.api.object_create(self.account,
self.container,
obj_name=self.path,
data="chunk")
meta, self.chunks = self.api.object_locate(self.account,
self.container, self.path)
self.version = meta['version']
self.content_id = meta['id']
self.chunk_method = meta['chunk_method']
def __init__(self, namespace, concurrency=50, error_file=None):
self.pool = GreenPool(concurrency)
self.error_file = error_file
if self.error_file:
f = open(self.error_file, 'a')
self.error_writer = csv.writer(f, delimiter=' ')
conf = {'namespace': namespace}
self.account_client = AccountClient(conf)
self.container_client = ContainerClient(conf)
self.blob_client = BlobClient()
self.accounts_checked = 0
self.containers_checked = 0
self.objects_checked = 0
self.chunks_checked = 0
self.account_not_found = 0
self.container_not_found = 0
self.object_not_found = 0
self.chunk_not_found = 0
self.account_exceptions = 0
self.container_exceptions = 0
self.object_exceptions = 0
self.chunk_exceptions = 0
self.list_cache = {}
self.running = {}
def init(self):
self.handlers = {
"plain": self._handle_rawx,
"ec": self._handle_rawx,
"backblaze": self._handle_b2,
}
self.blob_client = BlobClient(self.conf)
def __init__(self, tool, queue_workers, queue_reply):
super(ContentRepairerWorker, self).__init__(tool, queue_workers,
queue_reply)
self.chunk_operator = ChunkOperator(self.conf, logger=self.logger)
self.blob_client = BlobClient(self.conf)
self.container_client = ContainerClient(self.conf, logger=self.logger)
def _send_copy(self, targets, copies, fullpath):
headers = {"x-oio-chunk-meta-full-path": fullpath}
if not hasattr(self, "blob_client"):
from oio.blob.client import BlobClient
self.blob_client = BlobClient()
for t, c in zip(targets, copies):
self.blob_client.chunk_link(t, c, headers=headers).status
def setUp(self):
super(TestBlobAuditor, self).setUp()
self.container = random_str(16)
self.cid = cid_from_name(self.account, self.container)
self.path = random_str(16)
self.api = ObjectStorageApi(self.ns)
self.blob_client = BlobClient(self.conf)
self.api.container_create(self.account, self.container)
_, chunks = self.api.container.content_prepare(
self.account, self.container, self.path, 1)
services = self.conscience.all_services('rawx')
self.rawx_volumes = dict()
for rawx in services:
tags = rawx['tags']
service_id = tags.get('tag.service_id', None)
if service_id is None:
service_id = rawx['addr']
volume = tags.get('tag.vol', None)
self.rawx_volumes[service_id] = volume
self.api.object_create(
self.account, self.container, obj_name=self.path, data="chunk")
meta, self.chunks = self.api.object_locate(
self.account, self.container, self.path)
self.version = meta['version']
self.content_id = meta['id']
def setUp(self):
super(TestPlainContent, self).setUp()
if len(self.conf['services']['rawx']) < 4:
self.skipTest(
"Plain tests needs more than 3 rawx to run")
self.namespace = self.conf['namespace']
self.account = self.conf['account']
self.chunk_size = self.conf['chunk_size']
self.gridconf = {"namespace": self.namespace}
self.content_factory = ContentFactory(
self.gridconf, logger=self.logger)
self.container_client = ContainerClient(
self.gridconf, logger=self.logger)
self.blob_client = BlobClient(self.conf, logger=self.logger)
self.container_name = "TestPlainContent-%f" % time.time()
self.container_client.container_create(account=self.account,
reference=self.container_name)
self.container_id = cid_from_name(self.account,
self.container_name).upper()
self.content = "%s-%s" % (self.__class__.__name__, random_str(4))
self.stgpol = "SINGLE"
self.stgpol_twocopies = "TWOCOPIES"
self.stgpol_threecopies = "THREECOPIES"
def setUp(self):
super(TestBlobAuditorFunctional, self).setUp()
self.namespace = self.conf['namespace']
self.account = self.conf['account']
self.test_dir = self.conf['sds_path']
rawx_num, rawx_path, rawx_addr = self.get_service_url('rawx')
self.rawx = 'http://' + rawx_addr
self.h = hashlib.new('md5')
conf = {"namespace": self.namespace}
self.auditor = BlobAuditorWorker(conf, get_logger(None), None)
self.container_c = ContainerClient(conf)
self.blob_c = BlobClient()
self.ref = random_str(8)
self.container_c.container_create(self.account, self.ref)
self.url_rand = random_id(64)
self.data = random_str(1280)
self.h.update(self.data)
self.hash_rand = self.h.hexdigest().lower()
self.content = TestContent(random_str(6), len(self.data),
self.url_rand, 1)
self.content.id_container = cid_from_name(self.account,
self.ref).upper()
self.chunk = TestChunk(self.content.size, self.url_rand, 0,
self.hash_rand)
self.chunk_url = "%s/%s" % (self.rawx, self.chunk.id_chunk)
self.chunk_proxy = {
"hash": self.chunk.md5,
"pos": "0",
"size": self.chunk.size,
"url": self.chunk_url
}
chunk_meta = {
'content_path': self.content.path,
'container_id': self.content.id_container,
'chunk_method': 'plain/nb_copy=3',
'policy': 'TESTPOLICY',
'id': '0000',
'version': 1,
'chunk_id': self.chunk.id_chunk,
'chunk_pos': self.chunk.pos,
'chunk_hash': self.chunk.md5,
}
self.blob_c.chunk_put(self.chunk_url, chunk_meta, self.data)
self.chunk_path = self.test_dir + '/data/' + self.namespace + \
'-rawx-1/' + self.chunk.id_chunk[0:3] + "/" + self.chunk.id_chunk
self.bad_container_id = '0' * 64
def __init__(self, *args, **kwargs):
super(ContentReaperFilter, self).__init__(*args, **kwargs)
self.handlers = {
"plain": self._handle_rawx,
"ec": self._handle_rawx,
"backblaze": self._handle_b2,
}
self.blob_client = BlobClient()
def init(self):
self.handlers = {
"plain": self._handle_rawx,
"ec": self._handle_rawx,
"backblaze": self._handle_b2,
}
kwargs = {k: v for k, v in self.conf.items()
if k in URLLIB3_POOLMANAGER_KWARGS}
self.blob_client = BlobClient(self.conf, logger=self.logger, **kwargs)
self.chunk_concurrency = int(self.conf.get('concurrency', 3))
self.chunk_timeout = float(self.conf.get('timeout', 5.0))
def __init__(self,
conf,
container_client=None,
blob_client=None,
logger=None,
**kwargs):
self.conf = conf
self.logger = logger or get_logger(conf)
self.container_client = container_client or \
ContainerClient(conf, logger=self.logger, **kwargs)
self.blob_client = blob_client or BlobClient(conf, **kwargs)
def setUp(self):
super(TestRebuilderCrawler, self).setUp()
self.namespace = self.conf['namespace']
self.account = self.conf['account']
self.gridconf = {"namespace": self.namespace}
self.container_client = ContainerClient(self.gridconf)
self.blob_client = BlobClient()
self.container_name = "TestRebuilderCrawler%d" % int(time.time())
self.container_client.container_create(acct=self.account,
ref=self.container_name)
def setUp(self):
super(TestContentFactory, self).setUp()
self.namespace = self.conf['namespace']
self.chunk_size = self.conf['chunk_size']
self.gridconf = {"namespace": self.namespace}
self.content_factory = ContentFactory(self.gridconf)
self.container_name = "TestContentFactory%f" % time.time()
self.blob_client = BlobClient()
self.container_client = ContainerClient(self.gridconf)
self.container_client.container_create(acct=self.account,
ref=self.container_name)
self.container_id = cid_from_name(self.account,
self.container_name).upper()
def __init__(self,
namespace,
concurrency=50,
error_file=None,
rebuild_file=None,
full=True,
limit_listings=0,
request_attempts=1):
self.pool = GreenPool(concurrency)
self.error_file = error_file
self.full = bool(full)
# Optimisation for when we are only checking one object
# or one container.
# 0 -> do not limit
# 1 -> limit account listings (list of containers)
# 2 -> limit container listings (list of objects)
self.limit_listings = limit_listings
if self.error_file:
f = open(self.error_file, 'a')
self.error_writer = csv.writer(f, delimiter=' ')
self.rebuild_file = rebuild_file
if self.rebuild_file:
fd = open(self.rebuild_file, 'a')
self.rebuild_writer = csv.writer(fd, delimiter='|')
conf = {'namespace': namespace}
self.account_client = AccountClient(conf,
max_retries=request_attempts - 1)
self.container_client = ContainerClient(
conf,
max_retries=request_attempts - 1,
request_attempts=request_attempts)
self.blob_client = BlobClient(conf=conf)
self.accounts_checked = 0
self.containers_checked = 0
self.objects_checked = 0
self.chunks_checked = 0
self.account_not_found = 0
self.container_not_found = 0
self.object_not_found = 0
self.chunk_not_found = 0
self.account_exceptions = 0
self.container_exceptions = 0
self.object_exceptions = 0
self.chunk_exceptions = 0
self.list_cache = {}
self.running = {}
def setUp(self):
with mock.patch('oio.container.client.gen_headers', gen_headers_mock):
super(TestFilters, self).setUp()
self.account = self.conf['account']
self.namespace = self.conf['namespace']
self.chunk_size = self.conf['chunk_size']
self.gridconf = {'namespace': self.namespace}
self.content_factory = ContentFactory(self.gridconf)
self.container_name = 'TestFilter%f' % time.time()
self.blob_client = BlobClient()
self.container_client = ContainerClient(self.gridconf)
self.container_client.container_create(acct=self.account,
ref=self.container_name)
self.container_id = cid_from_name(self.account,
self.container_name).upper()
self.stgpol = "SINGLE"
def __init__(self, conf, job_params, logger=None):
super(RawxDecommissionTask, self).__init__(conf,
job_params,
logger=logger)
self.service_id = job_params['service_id']
self.rawx_timeout = job_params['rawx_timeout']
self.min_chunk_size = job_params['min_chunk_size']
self.max_chunk_size = job_params['max_chunk_size']
self.excluded_rawx = job_params['excluded_rawx']
self.blob_client = BlobClient(self.conf, logger=self.logger)
self.content_factory = ContentFactory(self.conf)
self.conscience_client = ConscienceClient(self.conf,
logger=self.logger)
self.fake_excluded_chunks = self._generate_fake_excluded_chunks(
self.excluded_rawx)
def setUp(self):
super(TestContentFactory, self).setUp()
self.namespace = self.conf['namespace']
self.chunk_size = self.conf['chunk_size']
self.gridconf = {"namespace": self.namespace}
self.content_factory = ContentFactory(self.gridconf)
self.container_name = "TestContentFactory%f" % time.time()
self.blob_client = BlobClient(conf=self.conf)
self.container_client = ContainerClient(self.gridconf)
self.container_client.container_create(account=self.account,
reference=self.container_name)
self.container_id = cid_from_name(self.account,
self.container_name).upper()
self.stgpol = "SINGLE"
self.stgpol_twocopies = "TWOCOPIES"
self.stgpol_threecopies = "THREECOPIES"
self.stgpol_ec = "EC"
def __init__(self, conf, container_id, metadata, chunks, storage_method):
self.conf = conf
self.container_id = container_id
self.metadata = metadata
self.chunks = ChunksHelper(chunks)
self.storage_method = storage_method
self.logger = get_logger(self.conf)
self.cs_client = ConscienceClient(conf)
self.blob_client = BlobClient()
self.container_client = ContainerClient(self.conf)
self.content_id = self.metadata["id"]
self.stgpol = self.metadata["policy"]
self.path = self.metadata["name"]
self.length = int(self.metadata["length"])
self.version = self.metadata["version"]
self.checksum = self.metadata["hash"]
self.mime_type = self.metadata["mime_type"]
self.chunk_method = self.metadata["chunk_method"]
def setUp(self):
super(TestDupContent, self).setUp()
if len(self.conf['rawx']) < 3:
self.skipTest("Not enough rawx. "
"Dup tests needs more than 2 rawx to run")
self.namespace = self.conf['namespace']
self.account = self.conf['account']
self.chunk_size = self.conf['chunk_size']
self.gridconf = {"namespace": self.namespace}
self.content_factory = ContentFactory(self.gridconf)
self.container_client = ContainerClient(self.gridconf)
self.blob_client = BlobClient()
self.container_name = "TestDupContent%f" % time.time()
self.container_client.container_create(acct=self.account,
ref=self.container_name)
self.container_id = cid_from_name(self.account,
self.container_name).upper()
def __init__(self, conf, container_id, metadata, chunks, stgpol_args):
self.conf = conf
self.container_id = container_id
self.metadata = metadata
self.chunks = ChunksHelper(chunks)
self.stgpol_args = stgpol_args
self.logger = get_logger(self.conf)
self.cs_client = ConscienceClient(conf)
self.container_client = ContainerClient(self.conf)
self.blob_client = BlobClient()
self.session = requests.Session()
self.content_id = self.metadata["id"]
self.stgpol_name = self.metadata["policy"]
self.path = self.metadata["name"]
self.length = int(self.metadata["length"])
self.version = self.metadata["version"]
self.hash = self.metadata["hash"]
self.mime_type = self.metadata["mime-type"]
self.chunk_method = self.metadata["chunk-method"]
def setUp(self):
super(TestBlobIndexer, self).setUp()
self.rdir_client = RdirClient(self.conf)
self.blob_client = BlobClient(self.conf)
_, self.rawx_path, rawx_addr, _ = \
self.get_service_url('rawx')
services = self.conscience.all_services('rawx')
self.rawx_id = None
for rawx in services:
if rawx_addr == rawx['addr']:
self.rawx_id = rawx['tags'].get('tag.service_id', None)
if self.rawx_id is None:
self.rawx_id = rawx_addr
conf = self.conf.copy()
conf['volume'] = self.rawx_path
self.blob_indexer = BlobIndexer(conf)
# clear rawx/rdir
chunk_files = paths_gen(self.rawx_path)
for chunk_file in chunk_files:
os.remove(chunk_file)
self.rdir_client.admin_clear(self.rawx_id, clear_all=True)
def setUp(self):
super(TestBlobAuditorFunctional, self).setUp()
self.namespace = self.conf['namespace']
self.account = self.conf['account']
self.ref = random_str(8)
_, rawx_loc, rawx_addr, rawx_uuid = \
self.get_service_url('rawx')
self.rawx_id = 'http://' + (rawx_uuid if rawx_uuid else rawx_addr)
self.auditor = BlobAuditorWorker(self.conf, get_logger(None), None)
self.container_client = ContainerClient(self.conf)
self.blob_client = BlobClient(conf=self.conf)
self.container_client.container_create(self.account, self.ref)
self.content = TestContent(self.account, self.ref)
self.chunk = TestChunk(self.content.cid, self.content.path,
self.content.version, self.storage_policy,
self.rawx_id, rawx_loc, self.content.size,
self.content.hash)
chunk_meta = {
'container_id': self.content.cid,
'content_path': self.content.path,
'version': self.content.version,
'id': self.content.id,
'full_path': self.content.fullpath,
'chunk_method': 'plain/nb_copy=3',
'policy': self.storage_policy,
'chunk_id': self.chunk.id,
'chunk_pos': self.chunk.pos,
'chunk_hash': self.chunk.metachunk_hash,
'chunk_size': self.chunk.metachunk_size,
'metachunk_hash': self.chunk.metachunk_hash,
'metachunk_size': self.chunk.metachunk_size,
'oio_version': OIO_VERSION
}
self.blob_client.chunk_put(self.chunk.url, chunk_meta,
self.content.data)
def __init__(self,
conf,
container_id,
metadata,
chunks,
storage_method,
account,
container_name,
container_client=None):
self.conf = conf
self.container_id = container_id
self.metadata = metadata
self.chunks = ChunksHelper(chunks)
self.storage_method = storage_method
self.logger = get_logger(self.conf)
self.blob_client = BlobClient()
self.container_client = (container_client
or ContainerClient(self.conf,
logger=self.logger))
# FIXME: all these may be properties
self.content_id = self.metadata["id"]
self.path = self.metadata["name"]
self.length = int(self.metadata["length"])
self.version = self.metadata["version"]
self.checksum = self.metadata["hash"]
self.chunk_method = self.metadata["chunk_method"]
self.account = account
self.container_name = container_name
if 'full_path' in self.metadata:
self.full_path = metadata['full_path']
else:
self.full_path = [
'{0}/{1}/{2}/{3}'.format(quote_plus(self.account),
quote_plus(self.container_name),
quote_plus(self.path), self.version)
]
def setUp(self):
super(TestECContent, self).setUp()
if len(self.conf['services']['rawx']) < 12:
self.skipTest("Not enough rawx. "
"EC tests needs at least 12 rawx to run")
self.namespace = self.conf['namespace']
self.account = self.conf['account']
self.chunk_size = self.conf['chunk_size']
self.gridconf = {"namespace": self.namespace}
self.content_factory = ContentFactory(self.gridconf)
self.container_client = ContainerClient(self.gridconf)
self.blob_client = BlobClient(self.conf)
self.container_name = "TestECContent%f" % time.time()
self.container_client.container_create(account=self.account,
reference=self.container_name)
self.container_id = cid_from_name(self.account,
self.container_name).upper()
self.content = "%s-%s" % (self.__class__.__name__, random_str(4))
self.stgpol = "EC"
self.size = 1024 * 1024 + 320
self.k = 6
self.m = 3
def setUp(self):
super(TestBlobAuditorFunctional, self).setUp()
self.namespace = self.conf['namespace']
self.account = self.conf['account']
self.test_dir = self.conf['sds_path']
self.chars = string.ascii_lowercase + string.ascii_uppercase +\
string.digits
self.chars_id = string.digits + 'ABCDEF'
self.rawx = 'http://' + self.conf["rawx"][0]['addr']
self.h = hashlib.new('md5')
conf = {"namespace": self.namespace}
self.auditor = BlobAuditorWorker(conf, get_logger(None), None)
self.container_c = ContainerClient(conf)
self.blob_c = BlobClient()
self.ref = rand_generator(self.chars, 8)
self.container_c.container_create(self.account, self.ref)
self.url_rand = rand_generator(self.chars_id, 64)
self.data = rand_generator(self.chars, 1280)
self.h.update(self.data)
self.hash_rand = self.h.hexdigest().lower()
self.content = TestContent(rand_generator(self.chars, 6),
len(self.data), self.url_rand, 1)
self.content.id_container = cid_from_name(self.account,
self.ref).upper()
self.chunk = TestChunk(self.content.size, self.url_rand, 0,
self.hash_rand)
self.chunk_url = "%s/%s" % (self.rawx, self.chunk.id_chunk)
self.chunk_proxy = {
"hash": self.chunk.md5,
"pos": "0",
"size": self.chunk.size,
"url": self.chunk_url
}
chunk_meta = {
'content_size': self.content.size,
'content_chunksnb': self.content.nb_chunks,
'content_path': self.content.path,
'content_cid': self.content.id_container,
'content_mimetype': 'application/octet-stream',
'content_chunkmethod': 'bytes',
'content_policy': 'TESTPOLICY',
'content_id': '0000',
'content_version': 1,
'chunk_id': self.chunk.id_chunk,
'chunk_pos': self.chunk.pos
}
self.blob_c.chunk_put(self.chunk_url, chunk_meta, self.data)
self.chunk_path = self.test_dir + '/data/' + self.namespace + \
'-rawx-1/' + self.chunk.id_chunk[0:3] + "/" + self.chunk.id_chunk
self.bad_container_id = '0' * 64
