def drop_privileges(user):
"""
Change the effective user of the current process, resets the current
directory to /.
"""
if os.geteuid() == 0:
groups = [g.gr_gid for g in grp.getgrall() if user in g.gr_mem]
os.setgroups(groups)
current_user = getuser()
if user != current_user:
try:
user_entry = pwd.getpwnam(user)
except KeyError as exc:
raise OioException("User %s does not exist (%s). Are you running "
"your namespace with another user name?" %
(user, exc))
try:
os.setgid(user_entry[3])
os.setuid(user_entry[2])
except OSError as exc:
raise OioException("Failed to switch uid to %s or gid to %s: %s" %
(user_entry[2], user_entry[3], exc))
os.environ['HOME'] = user_entry[5]
try:
os.setsid()
except OSError:
pass
os.chdir('/')
os.umask(0o22)
def _load_handler(self, chunk_method):
storage_method = STORAGE_METHODS.load(chunk_method)
handler = self.handlers.get(storage_method.type)
if not handler:
raise OioException("No handler found for chunk method [%s]" %
chunk_method)
return handler, storage_method
def _direct_request(self,
method,
url,
headers=None,
request_attempts=None,
**kwargs):
if not request_attempts:
request_attempts = self._request_attempts
if request_attempts <= 0:
raise OioException("Negative request attempts: %d" %
request_attempts)
if kwargs.get("autocreate"):
if not headers:
headers = dict()
headers["X-oio-action-mode"] = "autocreate"
kwargs = kwargs.copy()
kwargs.pop("autocreate")
for i in range(request_attempts):
try:
return super(ProxyClient,
self)._direct_request(method,
url,
headers=headers,
**kwargs)
except ServiceBusy:
if i >= request_attempts - 1:
raise
# retry with exponential backoff
ProxyClient._exp_sleep(i + 1)
def local_services(self):
resp, body = self._request('GET', '/list')
if resp.status_code == 200:
return body
else:
raise OioException("failed to get list of local services: %s" %
resp.text)
def all_services(self, type_, full=False, **kwargs):
"""
Get the list of all services of a specific type.
:param type_: the type of services to get (ex: 'rawx')
:type type_: `str`
:param full: whether to get all metrics for each service
:returns: the list of all services of the specified type.
:rtype: `list` of `dict` objects, each containing at least
- 'addr' (`str`),
- 'id' (`str`),
- 'score' (`int`),
- 'tags' (`dict`).
"""
params = {'type': type_}
if full:
params['full'] = '1'
resp, body = self._request('GET', '/list', params=params, **kwargs)
if resp.status == 200:
# TODO(FVE): do that in the proxy
for srv in body:
if 'id' not in srv:
srv_id = srv['tags'].get('tag.service_id', srv['addr'])
srv['id'] = srv_id
return body
else:
raise OioException("failed to get list of %s services: %s" %
(type_, resp.text))
def local_services(self):
url = self.endpoint.replace('conscience', 'local/list')
resp, body = self._direct_request('GET', url)
if resp.status == 200:
return body
else:
raise OioException("failed to get list of local services: %s" %
resp.text)
def next_instance(self, pool):
uri = self._make_uri('lb/choose')
params = {'pool': pool}
resp, body = self._request('GET', uri, params=params)
if resp.status_code == 200:
return body[0]
else:
raise OioException('ERROR while getting next instance %s' % pool)
def service_types(self):
params = {'what': 'types'}
resp, body = self._request('GET', '/info', params=params)
if resp.status == 200:
return body
else:
raise OioException("ERROR while getting services types: %s" %
resp.text)
def resolve(self, srv_type, service_id):
resp, body = self._request('GET', '/resolve',
params={'type': srv_type,
'service_id': service_id})
if resp.status == 200:
return body
else:
raise OioException("failed to resolve servie id %s: %s" %
(service_id, resp.text))
def all_services(self, type_):
uri = self._make_uri("conscience/list")
params = {'type': type_}
resp, body = self._request('GET', uri, params=params)
if resp.status_code == 200:
return body
else:
# FIXME: add resp error message
raise OioException("ERROR while getting list of %s services" %
type_)
def _get_streams(self):
for pos in range(len(self.meta_chunks)):
handler = BackblazeChunkDownloadHandler(self.sysmeta,
self.meta_chunks[pos],
0, 0, None,
self.backblaze_infos)
stream = handler.get_stream()
if not stream:
raise OioException("Error while downloading")
yield stream
def all_services(self, type_, full=False, **kwargs):
params = {'type': type_}
if full:
params['full'] = '1'
resp, body = self._request('GET', '/list', params=params, **kwargs)
if resp.status == 200:
return body
else:
raise OioException("failed to get list of %s services: %s" %
(type_, resp.text))
def assign_services(self, service_type, max_per_rdir=None, **kwargs):
all_services = self.cs.all_services(service_type, **kwargs)
all_rdir = self.cs.all_services('rdir', True, **kwargs)
if len(all_rdir) <= 0:
raise ServiceUnavailable("No rdir service found in %s" % self.ns)
by_id = {_make_id(self.ns, 'rdir', x['addr']): x for x in all_rdir}
errors = list()
for provider in all_services:
provider_id = provider['tags'].get('tag.service_id',
provider['addr'])
try:
resp = self.directory.list(RDIR_ACCT,
provider_id,
service_type='rdir',
**kwargs)
rdir_host = _filter_rdir_host(resp)
try:
provider['rdir'] = by_id[_make_id(self.ns, 'rdir',
rdir_host)]
except KeyError:
self.logger.warn("rdir %s linked to %s %s seems down",
rdir_host, service_type, provider_id)
except NotFound:
try:
rdir = self._smart_link_rdir(provider_id,
all_rdir,
service_type=service_type,
max_per_rdir=max_per_rdir,
**kwargs)
except OioException as exc:
self.logger.warn("Failed to link an rdir to %s %s: %s",
service_type, provider_id, exc)
errors.append((provider_id, exc))
continue
n_bases = by_id[rdir]['tags'].get("stat.opened_db_count", 0)
by_id[rdir]['tags']["stat.opened_db_count"] = n_bases + 1
provider['rdir'] = by_id[rdir]
except OioException as exc:
self.logger.warn(
"Failed to check rdir linked to %s %s "
"(thus won't try to make the link): %s", service_type,
provider_id, exc)
errors.append((provider_id, exc))
if errors:
# group_chunk_errors is flexible enough to accept service addresses
errors = group_chunk_errors(errors)
if len(errors) == 1:
err, addrs = errors.popitem()
oio_reraise(type(err), err, str(addrs))
else:
raise OioException('Several errors encountered: %s' % errors)
return all_services
def _delete(self, conn):
sysmeta = conn['sysmeta']
try_number = TRY_REQUEST_NUMBER
try:
conn['backblaze'].delete(self.backblaze_info['bucket_name'],
sysmeta)
except BackblazeException as b2e:
if try_number == 0:
raise OioException('backblaze delete error: %s'
% str(b2e))
try_number -= 1
def _delete(self, conn):
sysmeta = conn['sysmeta']
try_number = TRY_REQUEST_NUMBER
while True:
try:
conn['backblaze'].delete(self.backblaze_info['bucket_name'],
sysmeta)
break
except BackblazeException as b2e:
if try_number == 0:
raise OioException('backblaze delete error: %s' % str(b2e))
else:
eventlet.sleep(pow(2, TRY_REQUEST_NUMBER - try_number))
try_number -= 1
def next_instances(self, pool, size=None, **kwargs):
"""
Get the next service instances from the specified pool.
:keyword size: number of services to get
:type size: `int`
"""
params = {'type': pool}
if size is not None:
params['size'] = size
resp, body = self._request('GET', '/choose', params=params, **kwargs)
if resp.status == 200:
return body
else:
raise OioException('ERROR while getting next instance %s' % pool)
def next_instances(self, pool, **kwargs):
"""
Get the next service instances from the specified pool.
Available options:
- size: number of services to get
- slot: comma-separated list of slots to poll
"""
uri = self._make_uri('lb/choose')
params = {'type': pool}
params.update(kwargs)
resp, body = self._request('GET', uri, params=params)
if resp.status_code == 200:
return body
else:
raise OioException('ERROR while getting next instance %s' % pool)
def next_instances(self, pool, **kwargs):
"""
Get the next service instances from the specified pool.
:keyword size: number of services to get
:type size: `int`
:keyword slot: comma-separated list of slots to poll
:type slot: `str`
"""
params = {'type': pool}
params.update(kwargs)
resp, body = self._request('GET', '/choose', params=params)
if resp.status == 200:
return body
else:
raise OioException('ERROR while getting next instance %s' % pool)
def _make_stream(self, source):
result = None
data = None
for chunk in self.chunks:
self.meta['name'] = _get_name(chunk)
try:
try_number = TRY_REQUEST_NUMBER
data = source.download(self.backblaze_info['bucket_name'],
self.meta, self.headers)
except BackblazeException as b2e:
if try_number == 0:
raise OioException('backblaze download error: %s'
% str(b2e))
try_number -= 1
if data:
result = data
return result
def poll(self, pool, **kwargs):
"""
Get a set of services from a predefined pool.
:keyword avoid: service IDs that must be avoided
:type avoid: `list`
:keyword known: service IDs that are already known
:type known: `list`
"""
params = {'pool': pool}
ibody = dict()
ibody.update(kwargs)
resp, obody = self._request('POST', '/poll', params=params,
data=json.dumps(ibody))
if resp.status == 200:
return obody
else:
raise OioException("Failed to poll %s: %s" % (pool, resp.text))
def next_instances(self, pool, **kwargs):
"""
Get the next service instances from the specified pool.
Available options:
- size: number of services to get
- stgcls: storage class of the services
- tagk: name of the tag to be matched
- tagv: value of the tag to be matched (required if tagk specified)
"""
uri = self._make_uri('lb/choose')
params = {'type': pool}
params.update(kwargs)
resp, body = self._request('GET', uri, params=params)
if resp.status_code == 200:
return body
else:
raise OioException(
'ERROR while getting next instance %s' % pool)
def _upload_chunks(self, conn, size, sha1, md5, temp):
try_number = TRY_REQUEST_NUMBER
while True:
self.meta_chunk['size'] = size
try:
logger.debug("sha1: %s", sha1)
conn['backblaze'].upload(self.backblaze_info['bucket_name'],
self.sysmeta, temp, sha1)
break
except BackblazeException as b2e:
temp.seek(0)
if try_number == 0:
raise OioException('backblaze upload error: %s'
% str(b2e))
try_number -= 1
self.meta_chunk['hash'] = md5
return self.meta_chunk["size"], [self.meta_chunk]
def all_services(self, type_, full=False, **kwargs):
"""
Get the list of all services of a specific type.
:param type_: the type of services to get (ex: 'rawx')
:type type_: `str`
:param full: whether to get all metrics for each service
:returns: the list of all services of the specified type
:rtype: `list` of `dict`
"""
params = {'type': type_}
if full:
params['full'] = '1'
resp, body = self._request('GET', '/list', params=params, **kwargs)
if resp.status == 200:
return body
else:
raise OioException("failed to get list of %s services: %s"
% (type_, resp.text))
def _direct_request(self,
method,
url,
headers=None,
request_attempts=None,
**kwargs):
if not request_attempts:
request_attempts = self._request_attempts
if request_attempts <= 0:
raise OioException("Negative request attempts: %d" %
request_attempts)
if kwargs.get("autocreate"):
if not headers:
headers = dict()
headers[HEADER_PREFIX + "action-mode"] = "autocreate"
kwargs.pop("autocreate")
if kwargs.get("tls"):
headers = headers or dict()
headers[HEADER_PREFIX + "upgrade-to-tls"] = kwargs.pop("tls")
for i in range(request_attempts):
try:
return super(ProxyClient,
self)._direct_request(method,
url,
headers=headers,
**kwargs)
except ServiceBusy:
if i >= request_attempts - 1:
raise
# retry with exponential backoff
ProxyClient._exp_sleep(i + 1)
except Conflict:
if i > 0 and method == 'POST':
# We were retrying a POST operation, it's highly probable
# that the original operation succeeded after we timed
# out. So we consider this a success and don't raise
# the exception.
return None, None
raise
def _upload_chunks(self, conn, size, sha1, md5, temp):
try_number = TRY_REQUEST_NUMBER
while True:
self.meta_chunk[0]['size'] = size
try:
conn['backblaze'].upload(self.backblaze_info['bucket_name'],
self.sysmeta, temp, sha1)
break
except BackblazeException as b2e:
temp.seek(0)
if try_number == 0:
logger.debug('headers sent: %s' % str(b2e.headers_send))
raise OioException('backblaze upload error: %s' % str(b2e))
else:
sleep_time_default = pow(2,
TRY_REQUEST_NUMBER - try_number)
sleep = b2e.headers_received.get("Retry-After",
sleep_time_default)
eventlet.sleep(sleep)
try_number -= 1
self.meta_chunk[0]['hash'] = md5
return self.meta_chunk[0]["size"], self.meta_chunk
def __init__(self, conf, tool):
super(_DistributedDispatcher, self).__init__(conf, tool)
self.sending = False
# All available beanstalkd
conscience_client = ConscienceClient(self.conf)
all_beanstalkd = conscience_client.all_services('beanstalkd')
all_available_beanstalkd = dict()
for beanstalkd in all_beanstalkd:
if beanstalkd['score'] <= 0:
continue
all_available_beanstalkd[beanstalkd['addr']] = beanstalkd
if not all_available_beanstalkd:
raise OioException('No beanstalkd available')
# Beanstalkd workers
workers_tube = self.conf.get('distributed_beanstalkd_worker_tube') \
or self.tool.DEFAULT_DISTRIBUTED_BEANSTALKD_WORKER_TUBE
self.beanstalkd_workers = dict()
for _, beanstalkd in all_available_beanstalkd.items():
beanstalkd_worker_addr = beanstalkd['addr']
# If the tube exists,
# there should be a service that listens to this tube
tubes = Beanstalk.from_url('beanstalk://' +
beanstalkd_worker_addr).tubes()
if workers_tube not in tubes:
continue
beanstalkd_worker = BeanstalkdSender(beanstalkd_worker_addr,
workers_tube, self.logger)
self.beanstalkd_workers[beanstalkd_worker_addr] = beanstalkd_worker
self.logger.info(
'Beanstalkd %s using tube %s is selected as a worker',
beanstalkd_worker.addr, beanstalkd_worker.tube)
if not self.beanstalkd_workers:
raise OioException('No beanstalkd worker available')
# Beanstalkd reply
beanstalkd_reply = dict()
try:
local_services = conscience_client.local_services()
for local_service in local_services:
if local_service['type'] != 'beanstalkd':
continue
beanstalkd = all_available_beanstalkd.get(
local_service['addr'])
if beanstalkd is None:
continue
if beanstalkd_reply \
and beanstalkd_reply['score'] >= beanstalkd['score']:
continue
beanstalkd_reply = beanstalkd
except Exception as exc: # pylint: disable=broad-except
self.logger.warning(
'ERROR when searching for beanstalkd locally: %s', exc)
if not beanstalkd_reply:
self.logger.warn('No beanstalkd available locally')
try:
beanstalkd = conscience_client.next_instance('beanstalkd')
beanstalkd_reply = all_available_beanstalkd[beanstalkd['addr']]
except Exception as exc: # pylint: disable=broad-except
self.logger.warning('ERROR when searching for beanstalkd: %s',
exc)
beanstalkd_reply_addr = beanstalkd_reply['addr']
# If the tube exists, another service must have already used this tube
tube_reply = workers_tube + '.reply.' + str(time.time())
tubes = Beanstalk.from_url('beanstalk://' +
beanstalkd_reply_addr).tubes()
if tube_reply in tubes:
raise OioException('Beanstalkd %s using tube %s is already used')
self.beanstalkd_reply = BeanstalkdListener(beanstalkd_reply_addr,
tube_reply, self.logger)
self.logger.info(
'Beanstalkd %s using tube %s is selected for the replies',
self.beanstalkd_reply.addr, self.beanstalkd_reply.tube)
def __init__(self, conf, tool):
super(_DistributedDispatcher, self).__init__(conf, tool)
self.sending = None
self.max_items_per_second = int_value(
self.conf.get('items_per_second'),
self.tool.DEFAULT_ITEM_PER_SECOND)
# All available beanstalkd
conscience_client = ConscienceClient(self.conf)
all_beanstalkd = conscience_client.all_services('beanstalkd')
all_available_beanstalkd = dict()
for beanstalkd in all_beanstalkd:
if beanstalkd['score'] <= 0:
continue
all_available_beanstalkd[beanstalkd['addr']] = beanstalkd
if not all_available_beanstalkd:
raise OioException('No beanstalkd available')
# Beanstalkd workers
workers_tube = self.conf.get('distributed_beanstalkd_worker_tube') \
or self.tool.DEFAULT_DISTRIBUTED_BEANSTALKD_WORKER_TUBE
self.beanstalkd_workers = dict()
for beanstalkd in locate_tube(all_available_beanstalkd.values(),
workers_tube):
beanstalkd_worker = BeanstalkdSender(beanstalkd['addr'],
workers_tube, self.logger)
self.beanstalkd_workers[beanstalkd['addr']] = beanstalkd_worker
self.logger.info(
'Beanstalkd %s using tube %s is selected as a worker',
beanstalkd_worker.addr, beanstalkd_worker.tube)
if not self.beanstalkd_workers:
raise OioException('No beanstalkd worker available')
nb_workers = len(self.beanstalkd_workers)
if self.max_items_per_second > 0:
# Max 2 seconds in advance
queue_size_per_worker = self.max_items_per_second * 2 / nb_workers
else:
queue_size_per_worker = 64
for _, beanstalkd_worker in self.beanstalkd_workers.items():
beanstalkd_worker.low_limit = queue_size_per_worker / 2
beanstalkd_worker.high_limit = queue_size_per_worker
# Beanstalkd reply
beanstalkd_reply = dict()
try:
local_services = conscience_client.local_services()
for local_service in local_services:
if local_service['type'] != 'beanstalkd':
continue
beanstalkd = all_available_beanstalkd.get(
local_service['addr'])
if beanstalkd is None:
continue
if beanstalkd_reply \
and beanstalkd_reply['score'] >= beanstalkd['score']:
continue
beanstalkd_reply = beanstalkd
except Exception as exc: # pylint: disable=broad-except
self.logger.warning(
'ERROR when searching for beanstalkd locally: %s', exc)
if not beanstalkd_reply:
self.logger.warn('No beanstalkd available locally')
try:
beanstalkd = conscience_client.next_instance('beanstalkd')
beanstalkd_reply = all_available_beanstalkd[beanstalkd['addr']]
except Exception as exc: # pylint: disable=broad-except
self.logger.warning('ERROR when searching for beanstalkd: %s',
exc)
beanstalkd_reply_addr = beanstalkd_reply['addr']
# If the tube exists, another service must have already used this tube
tube_reply = workers_tube + '.reply.' + str(time.time())
tubes = Beanstalk.from_url('beanstalk://' +
beanstalkd_reply_addr).tubes()
if tube_reply in tubes:
raise OioException('Beanstalkd %s using tube %s is already used')
self.beanstalkd_reply = BeanstalkdListener(beanstalkd_reply_addr,
tube_reply, self.logger)
self.logger.info(
'Beanstalkd %s using tube %s is selected for the replies',
self.beanstalkd_reply.addr, self.beanstalkd_reply.tube)
def assign_services(self,
service_type,
max_per_rdir=None,
min_dist=None,
service_id=None,
reassign=False,
**kwargs):
"""
Assign an rdir service to all `service_type` servers that aren't
already assigned one.
:param max_per_rdir: Maximum number of services an rdir can handle.
:type max_per_rdir: `int`
:param min_dist: Minimum required distance between any service and
its assigned rdir service.
:type min_dist: `int`
:param service_id: Assign only this service ID.
:type service_id: `str`
:param reassign: Reassign an rdir service.
:type reassign: `bool`
:param dry_run: Display actions but do nothing.
:type dry_run: `bool`
:returns: The list of `service_type` services that were assigned
rdir services.
"""
all_services = self.cs.all_services(service_type, **kwargs)
if service_id:
for provider in all_services:
provider_id = provider['tags'].get('tag.service_id',
provider['addr'])
if service_id == provider_id:
break
else:
raise ValueError('%s isn\'t a %s' % (service_id, service_type))
all_services = [provider]
all_rdir = self.cs.all_services('rdir', True, **kwargs)
if len(all_rdir) <= 0:
raise ServiceUnavailable("No rdir service found in %s" % self.ns)
by_id = _build_dict_by_id(self.ns, all_rdir)
errors = list()
for provider in all_services:
provider_id = provider['tags'].get('tag.service_id',
provider['addr'])
try:
resp = self.directory.list(RDIR_ACCT,
provider_id,
service_type='rdir',
**kwargs)
rdir_host = _filter_rdir_host(resp)
try:
rdir = by_id[_make_id(self.ns, 'rdir', rdir_host)]
if reassign:
rdir['tags']['stat.opened_db_count'] = \
rdir['tags'].get('stat.opened_db_count', 0) - 1
# TODO(adu) Delete database
raise NotFound('Reassign an rdir services')
provider['rdir'] = rdir
except KeyError:
self.logger.warn("rdir %s linked to %s %s seems down",
rdir_host, service_type, provider_id)
if reassign:
raise NotFound('Reassign an rdir services')
except NotFound:
try:
rdir = self._smart_link_rdir(provider_id,
all_rdir,
service_type=service_type,
max_per_rdir=max_per_rdir,
min_dist=min_dist,
reassign=reassign,
**kwargs)
except OioException as exc:
self.logger.warn("Failed to link an rdir to %s %s: %s",
service_type, provider_id, exc)
errors.append((provider_id, exc))
continue
n_bases = by_id[rdir]['tags'].get("stat.opened_db_count", 0)
by_id[rdir]['tags']["stat.opened_db_count"] = n_bases + 1
provider['rdir'] = by_id[rdir]
except OioException as exc:
self.logger.warn(
"Failed to check rdir linked to %s %s "
"(thus won't try to make the link): %s", service_type,
provider_id, exc)
errors.append((provider_id, exc))
if errors:
# group_chunk_errors is flexible enough to accept service addresses
errors = group_chunk_errors(errors)
if len(errors) == 1:
err, addrs = errors.popitem()
oio_reraise(type(err), err, str(addrs))
else:
raise OioException('Several errors encountered: %s' % errors)
return all_services
def _stream_big_chunks(self, source, conn, temp):
max_chunk_size = conn['backblaze'].BACKBLAZE_MAX_CHUNK_SIZE
sha1_array = []
res = None
size, sha1, md5 = _read_to_temp(max_chunk_size, source,
self.checksum, temp)
# obligated to read max_chunk_size + 1 bytes
# if the size of the file is max_chunk_size
# backblaze will not take it because
# the upload part must have at least 2 parts
first_byte = source.read(1)
if not first_byte:
return self._upload_chunks(conn, size, sha1, md5, temp)
tries = TRY_REQUEST_NUMBER
while True:
try:
res = conn['backblaze'].upload_part_begin(
self.backblaze_info['bucket_name'], self.sysmeta)
break
except BackblazeException as b2e:
tries = tries - 1
if tries == 0:
raise OioException('Error at the beginning of upload: %s'
% str(b2e))
file_id = res['fileId']
part_num = 1
bytes_read = size + 1
tries = TRY_REQUEST_NUMBER
while True:
while True:
if bytes_read + max_chunk_size > self.meta_chunk['size']:
to_read = self.meta_chunk['size'] - bytes_read
else:
to_read = max_chunk_size
try:
res, sha1 = conn['backblaze'].upload_part(file_id, temp,
part_num, sha1)
break
except BackblazeException as b2e:
temp.seek(0)
tries = tries - 1
if tries == 0:
raise OioException('Error during upload: %s'
% str(b2e))
part_num += 1
sha1_array.append(sha1)
temp.seek(0)
temp.truncate(0)
size, sha1, md5 = _read_to_temp(to_read, source,
self.checksum, temp,
first_byte)
first_byte = None
bytes_read = bytes_read + size
if size == 0:
break
tries = TRY_REQUEST_NUMBER
while True:
try:
res = conn['backblaze'].upload_part_end(file_id,
sha1_array)
break
except BackblazeException as b2e:
tries = tries - 1
if tries == 0:
raise OioException('Error at the end of upload: %s'
% str(b2e))
self.meta_chunk['hash'] = md5
return bytes_read, [self.meta_chunk]