def delete(self, req, lock):
try:
volume = self.helper.volumes.get(self.id)
except NotFound:
raise HTTPNotFound("Cannot delete non-existant volume '%s'" %
self.id)
try:
# delete export in try block to avoid race
out = self.helper.exports.delete(self.id)
except NotFound:
# Might have recieved a duplicate delete request,
# but are still in the process of deletion
logger.debug("Requested deletion of '%s' but no export was "
"found" % self.id)
except ResourceBusy:
raise HTTPConflict("Cannot delete '%s' while export in "
"use" % self.id)
def callback():
self.helper.make_api_request('volumes', self.id,
data={'status': 'DELETED'})
# delete volume
try:
self.helper.cgroups.set_read_iops(volume, 0)
self.helper.cgroups.set_write_iops(volume, 0)
out = self.helper.volumes.delete(self.id, callback, lock)
volume['status'] = 'DELETING'
return Response(volume)
except NotFound:
raise HTTPNotFound("No volume named '%s'" % self.id)
def copy_volume(src_volume,
dest_volume,
block_size=4194304,
callback=None):
src, dest = None, None
try:
src = os.open(src_volume, os.O_RDONLY)
dest = os.open(dest_volume, os.O_WRONLY)
# Get the size of the source volume
size = os.lseek(src, 0, os.SEEK_END)
# Seek back to the beginning of the source device
os.lseek(src, 0, os.SEEK_SET)
for block in xrange(0, size, block_size):
if block % 100 == 0: # Every 1000 Blocks
percent = float(block) / size * 1000
logger.debug('cur_pos = %s (%d%%)' % (block, percent))
if callback:
callback(percent)
os.write(dest, os.read(src, block_size))
finally:
if dest:
try:
os.fsync(dest)
except OSError:
# Unit tests do not use a device that supports fsync()
pass
os.close(dest)
if src:
os.close(src)
def delete(self, req, lock):
try:
volume = self.helper.volumes.get(self.id)
except NotFound:
raise HTTPNotFound("Cannot delete non-existant volume '%s'" %
self.id)
try:
# delete export in try block to avoid race
out = self.helper.exports.delete(self.id)
except NotFound:
# Might have recieved a duplicate delete request,
# but are still in the process of deletion
logger.debug("Requested deletion of '%s' but no export was "
"found" % self.id)
except ResourceBusy:
raise HTTPConflict("Cannot delete '%s' while export in "
"use" % self.id)
def callback():
self.helper.make_api_request('volumes',
self.id,
data={'status': 'DELETED'})
# delete volume
try:
self.helper.cgroups.set_read_iops(volume, 0)
self.helper.cgroups.set_write_iops(volume, 0)
out = self.helper.volumes.delete(self.id, callback, lock)
volume['status'] = 'DELETING'
return Response(volume)
except NotFound:
raise HTTPNotFound("No volume named '%s'" % self.id)
def copy_volume(src_volume, dest_volume, block_size=4194304,
callback=None):
src, dest = None, None
try:
src = os.open(src_volume, os.O_RDONLY)
dest = os.open(dest_volume, os.O_WRONLY)
# Get the size of the source volume
size = os.lseek(src, 0, os.SEEK_END)
# Seek back to the beginning of the source device
os.lseek(src, 0, os.SEEK_SET)
for block in xrange(0, size, block_size):
if block % 1000 == 0: # Every 1000 Blocks
percent = float(block) / size * 100
logger.debug('cur_pos = %s (%d%%)' % (block, percent))
if callback:
callback(percent)
os.write(dest, os.read(src, block_size))
finally:
if dest:
try:
os.fsync(dest)
except OSError:
# Unit tests do not use a device that supports fsync()
pass
os.close(dest)
if src:
os.close(src)
def _mkfs_version(self):
args = ['/sbin/mkfs.ext4', '-V']
logger.debug("execute: %s" % args)
p = subprocess.Popen(args, close_fds=True, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, err = p.communicate()
logger.debug('returned: %s' % p.returncode)
if p.returncode:
raise ProcessError(' '.join(args), out, err, p.returncode)
return err.rstrip()
def _mkfs_version(self):
args = ['/sbin/mkfs.ext4', '-V']
logger.debug("execute: %s" % args)
p = subprocess.Popen(args, close_fds=True, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, err = p.communicate()
logger.debug('returned: %s' % p.returncode)
if p.returncode:
raise ProcessError(' '.join(args), out, err, p.returncode)
return err.rstrip()
def request(method, path, headers=None, data=None):
"""
How many times must I write this function before I just add requests
to our pip requires?
"""
req = urllib2.Request(path, headers=headers, data=data)
req.get_method = lambda *args: method
resp = urllib2.urlopen(req)
data = resp.read()
logger.debug("%s on %s succeeded with %s" %
(req.get_method(), req.get_full_url(), resp.getcode()))
if data:
return json.loads(data)
def request(method, path, headers=None, data=None):
"""
How many times must I write this function before I just add requests
to our pip requires?
"""
req = urllib2.Request(path, headers=headers, data=data)
req.get_method = lambda *args: method
resp = urllib2.urlopen(req)
data = resp.read()
logger.debug(
"%s on %s succeeded with %s" %
(req.get_method(), req.get_full_url(), resp.getcode()))
if data:
return json.loads(data)
def node_request(self, node, method, path, **kwargs):
url = "http://%s:%s%s" % (node.hostname, node.port, path)
headers = {"X-Request-Id": getattr(logger.local, "request_id", "lunr-%s" % uuid4())}
if method in ("GET", "HEAD", "DELETE"):
url += "?" + urlencode(kwargs)
req = Request(url, urlencode(kwargs), headers=headers)
req.get_method = lambda *args, **kwargs: method
try:
resp = urlopen(req, timeout=self.app.node_timeout)
logger.debug("%s on %s succeeded with %s" % (req.get_method(), req.get_full_url(), resp.getcode()))
return loads(resp.read())
except (socket.timeout, urllib2.HTTPError, urllib2.URLError, HTTPException), e:
raise NodeError(req, e)
def __init__(self, conf):
self.ietd_config = conf.string("export", "ietd_config", "/etc/iet/ietd.conf")
self.volume_group = conf.string("volume", "volume_group", "lunr-volume")
self.iqn_prefix = conf.string("export", "iqn_prefix", "iqn.2010-11.com.rackspace")
self.device_prefix = conf.string("export", "device_prefix", "/dev")
self.proc_iet_volume = conf.string("export", "proc_iet_volume", "/proc/net/iet/volume")
self.proc_iet_session = conf.string("export", "proc_iet_session", "/proc/net/iet/session")
self.initiators_allow = conf.string("export", "initiators_allow", "/etc/iet/initiators.allow")
self.default_allows = conf.string("export", "default_allows", "ALL")
logger.info("Setting export default_allows: %s" % self.default_allows)
subnets = conf.list("export", "allow_subnets", "0.0.0.0/0")
logger.debug("Setting export allow_subnets: %s" % subnets)
self.allow_subnets = []
for subnet in subnets:
if subnet:
self.allow_subnets.append(netaddr.IPNetwork(subnet))
self.run_dir = conf.string("storage", "run_dir", conf.path("run"))
def match(self, request):
route = self.urlmap.match(environ=request.environ)
if route is None:
raise HTTPNotFound("No route matched for path '%s'" % request.path)
logger.debug("Request path: %s matched: %s" %
(request.path, repr(route)))
try:
# Try to call the controller and action
controller = route['controller'](route, self)
return controller.__getattribute__(route['action'])
except AttributeError, e:
# Couldn't find the action on the controller
logger.debug(
'No action (%s) for controller (%s) Error: %s' %
(route.get('action', ''), route.get('controller', ''), e))
raise HTTPNotImplemented("No action for path '%s'" % request.path)
def match(self, request):
route = self.urlmap.match(environ=request.environ)
if route is None:
raise HTTPNotFound("No route matched for path '%s'" % request.path)
logger.debug("Request path: %s matched: %s" % (request.path,
repr(route)))
try:
# Try to call the controller and action
controller = route['controller'](route, self)
return controller.__getattribute__(route['action'])
except AttributeError, e:
# Couldn't find the action on the controller
logger.debug(
'No action (%s) for controller (%s) Error: %s' %
(route.get('action', ''), route.get('controller', ''), e))
raise HTTPNotImplemented("No action for path '%s'" % request.path)
def node_request(self, node, method, path, **kwargs):
url = 'http://%s:%s%s' % (node.hostname, node.port, path)
headers = {'X-Request-Id': getattr(logger.local, 'request_id',
'lunr-%s' % uuid4())}
if method in ('GET', 'HEAD', 'DELETE'):
url += '?' + urlencode(kwargs)
req = Request(url, urlencode(kwargs), headers=headers)
req.get_method = lambda *args, **kwargs: method
try:
resp = urlopen(req, timeout=self.app.node_timeout)
logger.debug(
"%s on %s succeeded with %s" %
(req.get_method(), req.get_full_url(), resp.getcode()))
return loads(resp.read())
except (socket.timeout, urllib2.HTTPError,
urllib2.URLError, HTTPException), e:
raise NodeError(req, e)
def execute(cmd, *args, **kwargs):
sudo = kwargs.pop('sudo', True)
if sudo:
args = ['sudo', cmd] + list(args)
else:
args = [cmd] + list(args)
for k, v in kwargs.items():
if k.startswith('_'):
k = k[1:]
args.append('--%s%s' % (k, format_value(v)))
logger.debug("execute: %s" % args)
p = subprocess.Popen(args, close_fds=True, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, err = p.communicate()
logger.debug('returned: %s' % p.returncode)
if p.returncode:
raise ProcessError(' '.join(args), out, err, p.returncode)
return out.rstrip()
def execute(cmd, *args, **kwargs):
sudo = kwargs.pop('sudo', True)
if sudo:
args = ['sudo', cmd] + list(args)
else:
args = [cmd] + list(args)
for k, v in kwargs.items():
if k.startswith('_'):
k = k[1:]
args.append('--%s%s' % (k, format_value(v)))
logger.debug("execute: %s" % args)
p = subprocess.Popen(args,
close_fds=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, err = p.communicate()
logger.debug('returned: %s' % p.returncode)
if p.returncode:
raise ProcessError(' '.join(args), out, err, p.returncode)
return out.rstrip()
def __init__(self, conf):
self.ietd_config = conf.string('export', 'ietd_config',
'/etc/iet/ietd.conf')
self.volume_group = conf.string('volume', 'volume_group',
'lunr-volume')
self.iqn_prefix = conf.string('export', 'iqn_prefix',
'iqn.2010-11.com.rackspace')
self.device_prefix = conf.string('export', 'device_prefix', '/dev')
self.proc_iet_volume = conf.string('export', 'proc_iet_volume',
'/proc/net/iet/volume')
self.proc_iet_session = conf.string('export', 'proc_iet_session',
'/proc/net/iet/session')
self.initiators_allow = conf.string('export', 'initiators_allow',
'/etc/iet/initiators.allow')
self.default_allows = conf.string('export', 'default_allows', 'ALL')
logger.info("Setting export default_allows: %s" % self.default_allows)
subnets = conf.list('export', 'allow_subnets', '0.0.0.0/0')
logger.debug("Setting export allow_subnets: %s" % subnets)
self.allow_subnets = []
for subnet in subnets:
if subnet:
self.allow_subnets.append(netaddr.IPNetwork(subnet))
self.run_dir = conf.string('storage', 'run_dir', conf.path('run'))
def __init__(self, conf):
self.ietd_config = conf.string('export', 'ietd_config',
'/etc/iet/ietd.conf')
self.volume_group = conf.string('volume', 'volume_group',
'lunr-volume')
self.iqn_prefix = conf.string('export', 'iqn_prefix',
'iqn.2010-11.com.rackspace')
self.device_prefix = conf.string('export', 'device_prefix', '/dev')
self.proc_iet_volume = conf.string('export', 'proc_iet_volume',
'/proc/net/iet/volume')
self.proc_iet_session = conf.string('export', 'proc_iet_session',
'/proc/net/iet/session')
self.initiators_allow = conf.string('export', 'initiators_allow',
'/etc/iet/initiators.allow')
self.default_allows = conf.string('export', 'default_allows', 'ALL')
logger.info("Setting export default_allows: %s" % self.default_allows)
subnets = conf.list('export', 'allow_subnets', '0.0.0.0/0')
logger.debug("Setting export allow_subnets: %s" % subnets)
self.allow_subnets = []
for subnet in subnets:
if subnet:
self.allow_subnets.append(netaddr.IPNetwork(subnet))
self.run_dir = conf.string('storage', 'run_dir', conf.path('run'))
def node_request(node_ip, node_port, method, path, **kwargs):
url = 'http://%s:%s%s' % (node_ip, node_port, path)
req_id = getattr(logger.local, 'request_id', None)
# This uuid() call can hang after a fork. libuuid reads from the wrong fd.
# This is a workaround for when node_request is used in the storage clone
# callback. We already have a request_id in that case.
if not req_id:
req_id = 'lunr-%s' % uuid4()
headers = {'X-Request-Id': req_id}
if method in ('GET', 'HEAD', 'DELETE'):
url += '?' + urlencode(kwargs)
req = Request(url, urlencode(kwargs), headers=headers)
req.get_method = lambda *args, **kwargs: method
try:
# FIXME. Magic number.
resp = urlopen(req, timeout=120)
logger.debug("%s on %s succeeded with %s" %
(req.get_method(), req.get_full_url(), resp.getcode()))
return loads(resp.read())
except (socket.timeout, HTTPError, URLError, HTTPException), e:
raise NodeError(req, e)
def node_request(node_ip, node_port, method, path, **kwargs):
url = 'http://%s:%s%s' % (node_ip, node_port, path)
req_id = getattr(logger.local, 'request_id', None)
# This uuid() call can hang after a fork. libuuid reads from the wrong fd.
# This is a workaround for when node_request is used in the storage clone
# callback. We already have a request_id in that case.
if not req_id:
req_id = 'lunr-%s' % uuid4()
headers = {'X-Request-Id': req_id}
if method in ('GET', 'HEAD', 'DELETE'):
url += '?' + urlencode(kwargs)
req = Request(url, urlencode(kwargs), headers=headers)
req.get_method = lambda *args, **kwargs: method
try:
# FIXME. Magic number.
resp = urlopen(req, timeout=120)
logger.debug(
"%s on %s succeeded with %s" %
(req.get_method(), req.get_full_url(), resp.getcode()))
return loads(resp.read())
except (socket.timeout, HTTPError, URLError, HTTPException), e:
raise NodeError(req, e)
def request(method, path, headers=None, data=None):
"""
How many times must I write this function before I just add requests
to our pip requires?
"""
req = urllib2.Request(path, headers=headers, data=data)
req.get_method = lambda *args: method
try:
resp = urllib2.urlopen(req)
data = resp.read()
except HTTPClientError.exceptions, e:
raise CinderError(req, e)
logger.debug(
"%s on %s succeeded with %s" %
(req.get_method(), req.get_full_url(), resp.getcode()))
if data:
return json.loads(data)
class CinderClient(object):
def __init__(self, username, password, auth_url, cinder_url,
tenant_id=None, rax_auth=True):
self.username = username
self.password = password
self.auth_url = auth_url.rstrip('/')
self.cinder_url = cinder_url.rstrip('/')
self.tenant_id = tenant_id
self._token = None
elif req.params.get('source_volume_id'):
source = self._validate_source_params(req)
# FIXME. Setting cgroups here would be silly, since we
# want a fast clone. How do we set them later?
# def callback():
# pass
# params['callback'] = callback
try:
self.helper.volumes.create(self.id, **params)
except AlreadyExists, e:
raise HTTPConflict(str(e))
volume = self.helper.volumes.get(self.id)
logger.debug('Created new volume %s to be clone of %s' %
(volume['id'], source['id']))
logger.debug('Creating export of new volume %s' % volume['id'])
try:
export = self.helper.exports.create(volume['id'])
except ServiceUnavailable:
self.helper.volumes.delete(volume['id'], lock=lock)
raise
# Tell other node to clone!
path = '/volumes/%s/clones/%s' % (source['id'], volume['id'])
node_params = {
'account': req.params.get('account', ''),
'iqn': export['name'],
'iscsi_ip': self.helper.storage_host,
'iscsi_port': self.helper.storage_port,
# First dirty method to close the export.
pass
def request(method, path, headers=None, data=None):
"""
How many times must I write this function before I just add requests
to our pip requires?
"""
req = urllib2.Request(path, headers=headers, data=data)
req.get_method = lambda *args: method
try:
resp = urllib2.urlopen(req)
data = resp.read()
except HTTPClientError.exceptions, e:
raise CinderError(req, e)
logger.debug("%s on %s succeeded with %s" %
(req.get_method(), req.get_full_url(), resp.getcode()))
if data:
return json.loads(data)
class CinderClient(object):
def __init__(self,
username,
password,
auth_url,
cinder_url,
tenant_id=None,
rax_auth=True):
self.username = username
self.password = password
self.auth_url = auth_url.rstrip('/')
elif req.params.get('source_volume_id'):
source = self._validate_source_params(req)
# FIXME. Setting cgroups here would be silly, since we
# want a fast clone. How do we set them later?
# def callback():
# pass
# params['callback'] = callback
try:
self.helper.volumes.create(self.id, **params)
except AlreadyExists, e:
raise HTTPConflict(str(e))
volume = self.helper.volumes.get(self.id)
logger.debug('Created new volume %s to be clone of %s'
% (volume['id'], source['id']))
logger.debug('Creating export of new volume %s' % volume['id'])
try:
export = self.helper.exports.create(volume['id'])
except ServiceUnavailable:
self.helper.volumes.delete(volume['id'], lock=lock)
raise
# Tell other node to clone!
path = '/volumes/%s/clones/%s' % (source['id'], volume['id'])
node_params = {
'account': req.params.get('account', ''),
'iqn': export['name'],
'iscsi_ip': self.helper.storage_host,
'iscsi_port': self.helper.storage_port,
# First dirty method to close the export.