def __init__(self, filter_, policy, **kwargs): super(Transition, self).__init__(filter_, **kwargs) self.policy = policy if self.lifecycle: from oio.content.factory import ContentFactory self.factory = ContentFactory(self.lifecycle.api.container.conf, self.lifecycle.api.container)
def __init__(self, conf, logger, volume, input_file=None, try_chunk_delete=False, beanstalkd_addr=None): self.conf = conf self.logger = logger or get_logger(conf) self.volume = volume self.run_time = 0 self.passes = 0 self.errors = 0 self.last_reported = 0 self.chunks_run_time = 0 self.bytes_running_time = 0 self.bytes_processed = 0 self.total_bytes_processed = 0 self.total_chunks_processed = 0 self.dry_run = true_value(conf.get('dry_run', False)) self.report_interval = int_value(conf.get('report_interval'), 3600) self.max_chunks_per_second = int_value(conf.get('chunks_per_second'), 30) self.max_bytes_per_second = int_value(conf.get('bytes_per_second'), 10000000) self.rdir_fetch_limit = int_value(conf.get('rdir_fetch_limit'), 100) self.allow_same_rawx = true_value(conf.get('allow_same_rawx')) self.input_file = input_file self.rdir_client = RdirClient(conf, logger=self.logger) self.content_factory = ContentFactory(conf) self.try_chunk_delete = try_chunk_delete self.beanstalkd_addr = beanstalkd_addr self.beanstalkd_tube = conf.get('beanstalkd_tube', 'rebuild') self.beanstalk = None
def __init__(self, conf, logger, volume): self.conf = conf self.logger = logger or get_logger(conf) self.volume = volume self.run_time = 0 self.passes = 0 self.errors = 0 self.last_reported = 0 self.last_usage_check = 0 self.chunks_run_time = 0 self.bytes_running_time = 0 self.bytes_processed = 0 self.total_bytes_processed = 0 self.total_chunks_processed = 0 self.usage_target = int_value( conf.get('usage_target'), 0) self.usage_check_interval = int_value( conf.get('usage_check_interval'), 3600) self.report_interval = int_value( conf.get('report_interval'), 3600) self.max_chunks_per_second = int_value( conf.get('chunks_per_second'), 30) self.max_bytes_per_second = int_value( conf.get('bytes_per_second'), 10000000) self.blob_client = BlobClient() self.container_client = ContainerClient(conf) self.content_factory = ContentFactory(conf)
def __init__(self, conf, logger=None, **kwargs): self.conf = conf self.logger = logger or get_logger(conf) volume = conf.get('volume') if not volume: raise ConfigurationException('No volume specified for converter') self.volume = volume self.namespace, self.volume_id = check_volume(self.volume) # cache self.name_by_cid = CacheDict() self.content_id_by_name = CacheDict() # client self.container_client = ContainerClient(conf, **kwargs) self.content_factory = ContentFactory(conf, self.container_client, logger=self.logger) # stats/logs self.errors = 0 self.passes = 0 self.total_chunks_processed = 0 self.start_time = 0 self.last_reported = 0 self.report_interval = int_value(conf.get('report_interval'), 3600) # speed self.chunks_run_time = 0 self.max_chunks_per_second = int_value(conf.get('chunks_per_second'), 30) # backup self.no_backup = true_value(conf.get('no_backup', False)) self.backup_dir = conf.get('backup_dir') or tempfile.gettempdir() self.backup_name = 'backup_%s_%f' \ % (self.volume_id, time.time()) # dry run self.dry_run = true_value(conf.get('dry_run', False))
def __init__(self, conf, logger, volume): self.conf = conf self.logger = logger or get_logger(conf) self.volume = volume self.namespace, self.address = check_volume(self.volume) self.running = False self.run_time = 0 self.passes = 0 self.errors = 0 self.last_reported = 0 self.last_usage_check = 0 self.chunks_run_time = 0 self.bytes_running_time = 0 self.bytes_processed = 0 self.total_bytes_processed = 0 self.total_chunks_processed = 0 self.concurrency = int_value(conf.get('concurrency'), 10) self.usage_target = int_value(conf.get('usage_target'), 0) self.usage_check_interval = int_value(conf.get('usage_check_interval'), 60) self.report_interval = int_value(conf.get('report_interval'), 3600) self.max_chunks_per_second = int_value(conf.get('chunks_per_second'), 30) self.limit = int_value(conf.get('limit'), 0) self.allow_links = true_value(conf.get('allow_links', True)) self.blob_client = BlobClient(conf) self.container_client = ContainerClient(conf, logger=self.logger) self.content_factory = ContentFactory( conf, container_client=self.container_client, blob_client=self.blob_client) self.excluded_rawx = \ [rawx for rawx in conf.get('excluded_rawx', '').split(',') if rawx] self.fake_excluded_chunks = self._generate_fake_excluded_chunks()
def setUp(self): super(TestPlainContent, self).setUp() if len(self.conf['services']['rawx']) < 4: self.skipTest( "Plain tests needs more than 3 rawx to run") self.namespace = self.conf['namespace'] self.account = self.conf['account'] self.chunk_size = self.conf['chunk_size'] self.gridconf = {"namespace": self.namespace} self.content_factory = ContentFactory( self.gridconf, logger=self.logger) self.container_client = ContainerClient( self.gridconf, logger=self.logger) self.blob_client = BlobClient(self.conf, logger=self.logger) self.container_name = "TestPlainContent-%f" % time.time() self.container_client.container_create(account=self.account, reference=self.container_name) self.container_id = cid_from_name(self.account, self.container_name).upper() self.content = "%s-%s" % (self.__class__.__name__, random_str(4)) self.stgpol = "SINGLE" self.stgpol_twocopies = "TWOCOPIES" self.stgpol_threecopies = "THREECOPIES"
def __init__(self, filter, policy_elt, **kwargs): super(Transition, self).__init__(filter, **kwargs) self.policy = policy_elt.text if self.lifecycle and self.lifecycle.api: from oio.content.factory import ContentFactory self.factory = ContentFactory(self.lifecycle.api.container.conf, self.lifecycle.api.container) else: self.factory = None
def __init__(self, rebuilder, try_chunk_delete=False, **kwargs): super(BlobRebuilderWorker, self).__init__(rebuilder, **kwargs) self.dry_run = true_value(self.rebuilder.conf.get('dry_run', False)) self.allow_same_rawx = true_value( self.rebuilder.conf.get('allow_same_rawx')) self.try_chunk_delete = try_chunk_delete self.rdir_client = self.rebuilder.rdir_client self.content_factory = ContentFactory(self.rebuilder.conf, logger=self.logger) self.sender = None
def __init__(self, conf, logger, volume, try_chunk_delete=False, **kwargs): super(BlobRebuilderWorker, self).__init__(conf, logger, **kwargs) self.volume = volume self.bytes_processed = 0 self.total_bytes_processed = 0 self.dry_run = true_value(conf.get('dry_run', False)) self.allow_same_rawx = true_value(conf.get('allow_same_rawx')) self.rdir_client = RdirClient(conf, logger=self.logger) self.content_factory = ContentFactory(conf, logger=self.logger) self.try_chunk_delete = try_chunk_delete
def __init__(self, policy=None, days=None, date=None, api=None): super(Transition, self).__init__(days=days, date=date, api=api) if policy is None: raise ValueError("'policy' must be specified") self.policy = policy if self.api: from oio.content.factory import ContentFactory self.factory = ContentFactory(api.container.conf, api.container) else: self.factory = None
def __init__(self, conf, logger, beanstalkd_addr, **kwargs): super(BlobImprover, self).__init__(conf, logger, volume=None, **kwargs) self.content_factory = ContentFactory(self.conf, logger=self.logger) beanstalkd_tube = self.conf.get('beanstalkd_tube', DEFAULT_IMPROVER_TUBE) self.listener = BeanstalkdListener(beanstalkd_addr, beanstalkd_tube, self.logger, **kwargs) self.sender = BeanstalkdSender(beanstalkd_addr, beanstalkd_tube, self.logger, **kwargs) self.retry_delay = int_value(self.conf.get('retry_delay'), 30) self.reqid_prefix = 'blob-impr-'
def setUp(self): super(TestContentFactory, self).setUp() self.namespace = self.conf['namespace'] self.chunk_size = self.conf['chunk_size'] self.gridconf = {"namespace": self.namespace} self.content_factory = ContentFactory(self.gridconf) self.container_name = "TestContentFactory%f" % time.time() self.container_client = ContainerClient(self.gridconf) self.container_client.container_create(acct=self.account, ref=self.container_name) self.container_id = cid_from_name(self.account, self.container_name).upper()
def setUp(self): super(TestStorageTierer, self).setUp() self.namespace = self.conf['namespace'] self.test_account = "test_storage_tiering_%f" % time.time() self.gridconf = {"namespace": self.namespace, "container_fetch_limit": 2, "content_fetch_limit": 2, "account": self.test_account, "outdated_threshold": 0, "new_policy": "EC"} self.content_factory = ContentFactory(self.gridconf) self.container_client = ContainerClient(self.gridconf) self._populate()
def setUp(self): super(TestFilters, self).setUp() self.account = self.conf['account'] self.namespace = self.conf['namespace'] self.chunk_size = self.conf['chunk_size'] self.gridconf = {'namespace': self.namespace} self.content_factory = ContentFactory(self.gridconf) self.container_name = 'TestFilter%f' % time.time() self.container_client = ContainerClient(self.gridconf, admin_mode=True) self.container_client.container_create(account=self.account, reference=self.container_name) self.container_id = cid_from_name(self.account, self.container_name).upper() self.stgpol = "SINGLE"
def setUp(self): with mock.patch('oio.container.client.gen_headers', gen_headers_mock): super(TestFilters, self).setUp() self.account = self.conf['account'] self.namespace = self.conf['namespace'] self.chunk_size = self.conf['chunk_size'] self.gridconf = {'namespace': self.namespace} self.content_factory = ContentFactory(self.gridconf) self.container_name = 'TestFilter%f' % time.time() self.blob_client = BlobClient() self.container_client = ContainerClient(self.gridconf) self.container_client.container_create(acct=self.account, ref=self.container_name) self.container_id = cid_from_name(self.account, self.container_name).upper() self.stgpol = "SINGLE"
def setUp(self): super(TestContentFactory, self).setUp() self.namespace = self.conf['namespace'] self.chunk_size = self.conf['chunk_size'] self.gridconf = {"namespace": self.namespace} self.content_factory = ContentFactory(self.gridconf) self.container_name = "TestContentFactory%f" % time.time() self.blob_client = BlobClient(conf=self.conf) self.container_client = ContainerClient(self.gridconf) self.container_client.container_create(account=self.account, reference=self.container_name) self.container_id = cid_from_name(self.account, self.container_name).upper() self.stgpol = "SINGLE" self.stgpol_twocopies = "TWOCOPIES" self.stgpol_threecopies = "THREECOPIES" self.stgpol_ec = "EC"
def __init__(self, conf, job_params, logger=None): super(RawxDecommissionTask, self).__init__(conf, job_params, logger=logger) self.service_id = job_params['service_id'] self.rawx_timeout = job_params['rawx_timeout'] self.min_chunk_size = job_params['min_chunk_size'] self.max_chunk_size = job_params['max_chunk_size'] self.excluded_rawx = job_params['excluded_rawx'] self.blob_client = BlobClient(self.conf, logger=self.logger) self.content_factory = ContentFactory(self.conf) self.conscience_client = ConscienceClient(self.conf, logger=self.logger) self.fake_excluded_chunks = self._generate_fake_excluded_chunks( self.excluded_rawx)
def setUp(self): super(TestDupContent, self).setUp() if len(self.conf['rawx']) < 3: self.skipTest("Not enough rawx. " "Dup tests needs more than 2 rawx to run") self.namespace = self.conf['namespace'] self.account = self.conf['account'] self.chunk_size = self.conf['chunk_size'] self.gridconf = {"namespace": self.namespace} self.content_factory = ContentFactory(self.gridconf) self.container_client = ContainerClient(self.gridconf) self.blob_client = BlobClient() self.container_name = "TestDupContent%f" % time.time() self.container_client.container_create(acct=self.account, ref=self.container_name) self.container_id = cid_from_name(self.account, self.container_name).upper()
def __init__(self, conf, logger, volume): self.conf = conf self.logger = logger or get_logger(conf) self.volume = volume self.run_time = 0 self.passes = 0 self.errors = 0 self.last_reported = 0 self.chunks_run_time = 0 self.bytes_running_time = 0 self.bytes_processed = 0 self.total_bytes_processed = 0 self.total_chunks_processed = 0 self.dry_run = true_value(conf.get('dry_run', False)) self.report_interval = int_value(conf.get('report_interval'), 3600) self.max_chunks_per_second = int_value(conf.get('chunks_per_second'), 30) self.max_bytes_per_second = int_value(conf.get('bytes_per_second'), 10000000) self.rdir_fetch_limit = int_value(conf.get('rdir_fetch_limit'), 100) self.rdir_client = RdirClient(conf) self.content_factory = ContentFactory(conf)
def __init__(self, conf, logger): self.conf = conf self.logger = logger self.account = conf[CONF_ACCOUNT] self.container_client = ContainerClient(self.conf) self.account_client = AccountClient(self.conf) self.content_factory = ContentFactory(self.conf) self.passes = 0 self.errors = 0 self.last_reported = 0 self.contents_run_time = 0 self.total_contents_processed = 0 self.report_interval = int_value(conf.get('report_interval'), 3600) self.max_contents_per_second = int_value( conf.get('contents_per_second'), 30) self.container_fetch_limit = int_value( conf.get('container_fetch_limit'), 100) self.content_fetch_limit = int_value(conf.get('content_fetch_limit'), 100) self.outdated_threshold = int_value(conf.get(CONF_OUTDATED_THRESHOLD), 9999999999) self.new_policy = conf.get(CONF_NEW_POLICY)
def setUp(self): super(TestECContent, self).setUp() if len(self.conf['services']['rawx']) < 12: self.skipTest("Not enough rawx. " "EC tests needs at least 12 rawx to run") self.namespace = self.conf['namespace'] self.account = self.conf['account'] self.chunk_size = self.conf['chunk_size'] self.gridconf = {"namespace": self.namespace} self.content_factory = ContentFactory(self.gridconf) self.container_client = ContainerClient(self.gridconf) self.blob_client = BlobClient(self.conf) self.container_name = "TestECContent%f" % time.time() self.container_client.container_create(account=self.account, reference=self.container_name) self.container_id = cid_from_name(self.account, self.container_name).upper() self.content = "%s-%s" % (self.__class__.__name__, random_str(4)) self.stgpol = "EC" self.size = 1024 * 1024 + 320 self.k = 6 self.m = 3
def __init__(self, conf, logger=None): self.conf = conf self.logger = logger or get_logger(conf) self.rdir_client = RdirClient(conf, logger=self.logger) self.content_factory = ContentFactory(conf, logger=self.logger)