def run(self):
self.pool = ThreadPool(processes=self.SYNC_WORKERS)
with self.dbconnect() as con:
cur = con.cursor()
cur.execute('''CREATE TABLE IF NOT EXISTS zvol_calls(
zvol TEXT PRIMARY KEY NOT NULL,
reply_to TEXT NOT NULL,
time INT NOT NULL)''')
cur.execute('''CREATE TABLE IF NOT EXISTS zvols(
zvol TEXT PRIMARY KEY NOT NULL,
zpool TEXT,
iscsi_target TEXT UNIQUE,
remotehost TEXT,
remotepool TEXT,
sync BOOLEAN)''')
cur.execute('''CREATE TABLE IF NOT EXISTS zvolattrs(
zvol TEXT PRIMARY KEY NOT NULL,
frequency INT DEFAULT NULL,
nextsync INT DEFAULT NULL,
downloadspeed INT DEFAULT NULL,
uploadspeed INT DEFAULT NULL)''')
cur.execute('''CREATE TABLE IF NOT EXISTS sync_queue(
zvol TEXT PRIMARY KEY NOT NULL,
zpool TEXT NOT NULL,
remotehost TEXT,
remotepool TEXT,
is_sending BOOLEAN,
is_delete_remote BOOLEAN,
time INT)''')
cur.execute('''CREATE TABLE IF NOT EXISTS globals(
attr TEXT PRIMARY KEY NOT NULL,
value TEXT)''')
cur.execute('DELETE FROM sync_queue')
con.commit()
# Record all parameters in the configuration file in globals
for key in self.nc.DATA.keys():
self.setAttr(key, self.nc.DATA[key])
if self.getAttr('frequency') is None:
self.setAttr('frequency', self.SYNC_PULL_DEFAULT)
self.queue_connector = RabbitMQCommonClient('rocks.vm-manage',
'direct', "img-storage", "img-storage",
self.process_message, lambda a:
self.startup(),
routing_key=self.nc.NODE_NAME,
ssl = True,
ssl_options = self.ssl_options,
encryption = self.use_encryption,
frontend = self.nc.FRONTEND_NAME,
secur_server = self.secur_server
)
self.queue_connector.run()
def run(self):
self.pool = ThreadPool(processes=self.SYNC_WORKERS)
with sqlite3.connect(self.SQLITE_DB) as con:
cur = con.cursor()
cur.execute('''CREATE TABLE IF NOT EXISTS zvol_calls(
zvol TEXT PRIMARY KEY NOT NULL,
reply_to TEXT NOT NULL,
time INT NOT NULL)''')
cur.execute('''CREATE TABLE IF NOT EXISTS zvols(
zvol TEXT PRIMARY KEY NOT NULL,
zpool TEXT,
iscsi_target TEXT UNIQUE,
remotehost TEXT)''')
cur.execute('''CREATE TABLE IF NOT EXISTS sync_queue(
zvol TEXT PRIMARY KEY NOT NULL,
zpool TEXT NOT NULL,
remotehost TEXT,
is_sending BOOLEAN,
is_delete_remote BOOLEAN,
time INT)''')
con.commit()
self.queue_connector = RabbitMQCommonClient('rocks.vm-manage',
'direct', "img-storage", "img-storage",
self.process_message, lambda a: \
self.startup(),
routing_key=NodeConfig.NODE_NAME)
self.queue_connector.run()
def run(self):
with sqlite3.connect(self.SQLITE_DB) as con:
cur = con.cursor()
cur.execute('CREATE TABLE IF NOT EXISTS zvol_calls(zvol TEXT PRIMARY KEY NOT NULL, reply_to TEXT NOT NULL, time INT NOT NULL)')
cur.execute('CREATE TABLE IF NOT EXISTS zvols(zvol TEXT PRIMARY KEY NOT NULL, iscsi_target TEXT UNIQUE, hosting TEXT)')
con.commit()
self.queue_connector = RabbitMQCommonClient('rocks.vm-manage', 'direct', self.process_message)
self.queue_connector.run()
def run(self):
self.pool = ThreadPool(processes=self.SYNC_WORKERS)
with self.dbconnect() as con:
cur = con.cursor()
cur.execute('''CREATE TABLE IF NOT EXISTS zvol_calls(
zvol TEXT PRIMARY KEY NOT NULL,
reply_to TEXT NOT NULL,
time INT NOT NULL)''')
cur.execute('''CREATE TABLE IF NOT EXISTS zvols(
zvol TEXT PRIMARY KEY NOT NULL,
zpool TEXT,
iscsi_target TEXT UNIQUE,
remotehost TEXT,
remotepool TEXT,
sync BOOLEAN)''')
cur.execute('''CREATE TABLE IF NOT EXISTS zvolattrs(
zvol TEXT PRIMARY KEY NOT NULL,
frequency INT DEFAULT NULL,
nextsync INT DEFAULT NULL,
downloadspeed INT DEFAULT NULL,
uploadspeed INT DEFAULT NULL)''')
cur.execute('''CREATE TABLE IF NOT EXISTS sync_queue(
zvol TEXT PRIMARY KEY NOT NULL,
zpool TEXT NOT NULL,
remotehost TEXT,
remotepool TEXT,
is_sending BOOLEAN,
is_delete_remote BOOLEAN,
time INT)''')
cur.execute('''CREATE TABLE IF NOT EXISTS globals(
attr TEXT PRIMARY KEY NOT NULL,
value TEXT)''')
cur.execute('DELETE FROM sync_queue')
con.commit()
# Record all parameters in the configuration file in globals
for key in self.nc.DATA.keys():
self.setAttr(key, self.nc.DATA[key])
if self.getAttr('frequency') is None:
self.setAttr('frequency', self.SYNC_PULL_DEFAULT)
self.queue_connector = RabbitMQCommonClient('rocks.vm-manage',
'direct', "img-storage", "img-storage",
self.process_message, lambda a:
self.startup(),
routing_key=self.nc.NODE_NAME)
self.queue_connector.run()
class NasDaemon:
def __init__(self):
# FIXME: user should be configurable at startup
self.imgUser = '******'
self.stdin_path = '/dev/null'
self.stdout_path = '/tmp/out.log'
self.stderr_path = '/tmp/err.log'
self.pidfile_path = '/var/run/img-storage-nas.pid'
self.pidfile_timeout = 5
self.function_dict = {
'map_zvol': self.map_zvol,
'unmap_zvol': self.unmap_zvol,
'zvol_mapped': self.zvol_mapped,
'zvol_unmapped': self.zvol_unmapped,
'list_zvols': self.list_zvols,
'del_zvol': self.del_zvol,
'get_zvol_attrs': self.get_zvol_attrs,
'set_zvol_attrs': self.set_zvol_attrs,
'get_attrs': self.get_attrs,
'set_attrs': self.set_attrs,
'del_attrs': self.del_attrs,
'zvol_synced': self.zvol_synced,
}
self.nc = NodeConfig.NodeConfig()
self.SQLITE_DB = '/opt/rocks/var/img_storage.db'
self.NODE_NAME = self.nc.NODE_NAME
self.ib_net = self.nc.SYNC_NETWORK
self.sync_result = None
self.results = {}
if self.nc.IMG_SYNC_WORKERS:
self.SYNC_WORKERS = int(self.nc.IMG_SYNC_WORKERS)
else:
self.SYNC_WORKERS = 5
self.SYNC_CHECK_TIMEOUT = 10
# try once/minute to add new jobs to the sync queue
self.SYNC_PULL_TIMEOUT = 60
# the default SYNC_PULL is 5 minutes
self.SYNC_PULL_DEFAULT = 60 * 5
self.logger = \
logging.getLogger('imgstorage.imgstoragenas.NasDaemon')
self.ZVOLATTRS = ['frequency', 'nextsync',
'downloadspeed', 'uploadspeed']
def dbconnect(self):
""" connect to sqlite3 database, turn on foreign constraints """
con = sqlite3.connect(self.SQLITE_DB)
cur = con.cursor()
cur.execute('PRAGMA foreign_keys=ON')
return con
def getZvolAttr(self, zvol, attr=None):
""" Return a single named attribute for a zvol, or a dictionary of
attributes if attr=None """
rval = None
with self.dbconnect() as con:
cur = con.cursor()
if attr is not None:
cur.execute("SELECT %s FROM zvolattrs WHERE zvol='%s'" %
(attr, zvol))
row = cur.fetchone()
if row != None:
rval = row[0]
else:
cur.execute("SELECT * from zvolattrs WHERE zvol='%s'" % zvol)
row = cur.fetchone()
if row != None:
rval = dict((cur.description[i][0], value)
for (i, value) in enumerate(row))
return rval
def setZvolAttr(self, zvol, attr, value=None):
""" Set a single named attribute for a zvol. Set to Null of value is None """
with self.dbconnect() as con:
cur = con.cursor()
cur.execute(
'SELECT count(*) FROM zvolattrs WHERE zvol = ?', [zvol])
if cur.fetchone()[0] == 0:
cur.execute('INSERT INTO zvolattrs(zvol) VALUES(?)', [zvol])
if value is None:
setStmt = "SET %s=NULL" % attr
elif isinstance(value, int):
setStmt = "SET %s=%d" % (attr, value)
else:
setStmt = "SET %s='%s'" % (attr, value)
cur.execute(""" UPDATE zvolattrs %s WHERE zvol='%s'""" %
(setStmt, zvol))
con.commit()
def getAttr(self, attr=None):
""" Return a single named global attribute or a dictionary of
all attributes if attr=None. All values are type string """
rval = None
with self.dbconnect() as con:
cur = con.cursor()
if attr is not None:
cur.execute("SELECT value FROM globals WHERE attr='%s'" %
(attr))
row = cur.fetchone()
if row != None:
rval = row[0]
else:
cur.execute("SELECT attr,value FROM globals")
rval = {}
for row in cur.fetchall():
attr, value = row
rval[attr] = value
return rval
def setAttr(self, attr, value=None):
""" Set a single named attribute.Set to Null of value is None """
with self.dbconnect() as con:
if value is not None:
value = str(value)
cur = con.cursor()
cur.execute('''INSERT OR REPLACE INTO globals(attr,value)
VALUES(?,?)''', [attr, value] )
con.commit()
def deleteAttr(self, attr):
""" delete single named attribute """
with self.dbconnect() as con:
cur = con.cursor()
cur.execute('''DELETE FROM globals WHERE attr="%s"''' % attr )
con.commit()
# Attribute get/set messages
def get_zvol_attrs(self, message, props):
# input: get_zvol_attr messages
# output: get_zvol_attr message --> requestor
zvol = message['zvol']
try:
attrs = self.getZvolAttr(zvol)
attrs['zvol'] = zvol
except Exception as err:
self.failAction(props.reply_to, 'get_zvol_attrs', str(err))
return
reply = json.dumps({'action': 'get_zvol_attrs', 'status': 'success',
'body': attrs})
self.queue_connector.publish_message(reply, exchange='',
routing_key=props.reply_to)
def set_zvol_attrs(self, message, props):
# input: set_zvol_attr message
# output: set_zvol_attr message --> requestor
# state updates: zvols attr table
zvol = message['zvol']
try:
for k in message.keys():
if k in self.ZVOLATTRS:
self.setZvolAttr(zvol, k, message[k])
except Exception as err:
self.failAction(props.reply_to, 'set_zvol_attrs', str(err))
return
reply = json.dumps({'action': 'set_zvol_attrs', 'status': 'success'})
self.queue_connector.publish_message(reply, exchange='',
routing_key=props.reply_to)
def get_attrs(self, message, props):
# input: get_attrs messages
# output: get_attrs message --> requestor
try:
attrs = self.getAttr()
except Exception as err:
self.failAction(props.reply_to, 'get_attrs', str(err))
return
reply = json.dumps({'action': 'get_attrs', 'status': 'success',
'attrs': attrs})
self.queue_connector.publish_message(reply, exchange='',
routing_key=props.reply_to)
def set_attrs(self, message, props):
# input: set_attrs message
# output: set_attrs message --> requestor
# state updates: attrs table
try:
attrs = message['attrs']
for k in attrs.keys():
self.setAttr(k, attrs[k])
except Exception as err:
self.failAction(props.reply_to, 'set_attrs', str(err))
return
reply = json.dumps({'action': 'set_attrs', 'status': 'success'})
self.queue_connector.publish_message(reply, exchange='',
routing_key=props.reply_to)
def del_attrs(self, message, props):
# input: del_attrs message
# output: del_attrs message --> requestor
# state updates: attrs table
try:
for attr in message['attrs']:
self.deleteAttr(attr)
except Exception as err:
self.failAction(props.reply_to, 'del_attrs', str(err))
return
reply = json.dumps({'action': 'del_attrs', 'status': 'success'})
self.queue_connector.publish_message(reply, exchange='',
routing_key=props.reply_to)
############ Main Run Method ###################
def run(self):
self.pool = ThreadPool(processes=self.SYNC_WORKERS)
with self.dbconnect() as con:
cur = con.cursor()
cur.execute('''CREATE TABLE IF NOT EXISTS zvol_calls(
zvol TEXT PRIMARY KEY NOT NULL,
reply_to TEXT NOT NULL,
time INT NOT NULL)''')
cur.execute('''CREATE TABLE IF NOT EXISTS zvols(
zvol TEXT PRIMARY KEY NOT NULL,
zpool TEXT,
iscsi_target TEXT UNIQUE,
remotehost TEXT,
remotepool TEXT,
sync BOOLEAN)''')
cur.execute('''CREATE TABLE IF NOT EXISTS zvolattrs(
zvol TEXT PRIMARY KEY NOT NULL,
frequency INT DEFAULT NULL,
nextsync INT DEFAULT NULL,
downloadspeed INT DEFAULT NULL,
uploadspeed INT DEFAULT NULL)''')
cur.execute('''CREATE TABLE IF NOT EXISTS sync_queue(
zvol TEXT PRIMARY KEY NOT NULL,
zpool TEXT NOT NULL,
remotehost TEXT,
remotepool TEXT,
is_sending BOOLEAN,
is_delete_remote BOOLEAN,
time INT)''')
cur.execute('''CREATE TABLE IF NOT EXISTS globals(
attr TEXT PRIMARY KEY NOT NULL,
value TEXT)''')
cur.execute('DELETE FROM sync_queue')
con.commit()
# Record all parameters in the configuration file in globals
for key in self.nc.DATA.keys():
self.setAttr(key, self.nc.DATA[key])
if self.getAttr('frequency') is None:
self.setAttr('frequency', self.SYNC_PULL_DEFAULT)
self.queue_connector = RabbitMQCommonClient('rocks.vm-manage',
'direct', "img-storage", "img-storage",
self.process_message, lambda a:
self.startup(),
routing_key=self.nc.NODE_NAME)
self.queue_connector.run()
def failAction(
self,
routing_key,
action,
error_message,
):
if routing_key != None and action != None:
self.queue_connector.publish_message(json.dumps({'action': action,
'status': 'error', 'error': error_message}),
exchange='', routing_key=routing_key)
self.logger.error('Failed %s: %s' % (action, error_message))
def startup(self):
self.schedule_zvols_pull()
self.schedule_next_sync()
@coroutine
def map_zvol(self, message, props):
# input: map_zvol message
# output: map_zvol message --> remotehost
# state updates: zvol table
remotehost = message['remotehost']
remotepool = message['remotepool']
zpool_name = message['zpool']
zvol_name = message['zvol']
sync = message['sync']
# print "XXX map_zvol (message): ", message
self.logger.debug('Setting zvol %s' % zvol_name)
with sqlite3.connect(self.SQLITE_DB) as con:
cur = con.cursor()
try:
self.lock_zvol(zvol_name, props.reply_to)
cur.execute(
'SELECT count(*) FROM zvols WHERE zvol = ?', [zvol_name])
volume = "%s/%s" % (zpool_name, zvol_name)
if cur.fetchone()[0] == 0:
""" Create a zvol, if it doesn't already exist """
# check if volume already exists
# print 'XXX checking if zvol %s exists' % volume
self.logger.debug('checking if zvol %s exists' % volume)
rcode = subprocess.call(["zfs", "list", volume])
# print 'XXX check complete (%s)' % volume
self.logger.debug('check complete (%s)' % volume)
if rcode != 0:
# create the zfs FS
yield runCommandBackground(zfs_create
+ ['-V', '%sgb' % message['size'], volume])
self.logger.debug('Created new zvol %s' % volume)
else:
self.logger.debug('Vol %s exists' % volume)
# Record the creation of the volume
cur.execute('''INSERT OR REPLACE INTO
zvols(zvol,zpool,iscsi_target,remotehost,remotepool,sync)
VALUES (?,?,?,?,?,?) '''
, (zvol_name, zpool_name, None, None, None, False))
con.commit()
# print 'XXX Created new zvol %s' % volume
self.logger.debug('Created new zvol %s' % volume)
cur.execute(
'SELECT remotehost FROM zvols WHERE zvol = ?', [zvol_name])
row = cur.fetchone()
if row != None and row[0] != None: # zvol is mapped
raise ActionError(
'Error when mapping zvol: already mapped')
ip = None
use_ib = False
if self.ib_net:
try:
ip = socket.gethostbyname('%s.%s'
% (remotehost, self.ib_net))
use_ib = True
except:
pass
if not use_ib:
try:
ip = socket.gethostbyname(remotehost)
except:
raise ActionError('Host %s is unknown'
% remotehost)
iscsi_target = ''
(out, err) = (yield runCommandBackground([
'/opt/rocks/bin/tgt-setup-lun-lock',
'-n',
zvol_name,
'-d',
'/dev/%s/%s' % (zpool_name, zvol_name),
ip,
]))
for line in out:
if 'Creating new target' in line:
start = 'Creating new target (name='.__len__()
iscsi_target = line[start:line.index(',')]
# print 'XXX Mapped %s to iscsi target %s' % (zvol_name,
# iscsi_target)
self.logger.debug('Mapped %s to iscsi target %s'
% (zvol_name, iscsi_target))
# Update the Zvols table with the target, remote and sync
# attributes
cur.execute('''INSERT OR REPLACE INTO
zvols(zvol,zpool,iscsi_target,remotehost, remotepool,sync)
VALUES (?,?,?,?,?,?) '''
, (zvol_name, zpool_name, iscsi_target, remotehost, remotepool, sync))
con.commit()
# print 'XXX iscsi target %s inserted into DB' % iscsi_target
def failDeliver(
target,
zvol,
reply_to,
remotehost,
):
self.detach_target(target, True)
self.failAction(props.reply_to, 'zvol_mapped',
'Compute node %s is unavailable'
% remotehost)
self.release_zvol(zvol_name)
# Send a map_zvol message to remotehost
# XXX: Should rename this message
self.queue_connector.publish_message(json.dumps({
'action': 'map_zvol',
'target': iscsi_target,
'nas': ('%s.%s' % (self.NODE_NAME,
self.ib_net) if use_ib else self.NODE_NAME),
'size': message['size'],
'zvol': zvol_name,
'sync': sync,
'remotehost': remotehost,
'remotepool': remotepool,
}), remotehost, self.NODE_NAME, on_fail=lambda:
failDeliver(iscsi_target, zvol_name,
props.reply_to, remotehost))
self.logger.debug('Setting iscsi %s sent'
% iscsi_target)
except ActionError, err:
if not isinstance(err, ZvolBusyActionError):
self.release_zvol(zvol_name)
self.failAction(props.reply_to, 'zvol_mapped', str(err))
class NasDaemon:
def __init__(self):
# FIXME: user should be configurable at startup
self.imgUser = '******'
self.stdin_path = '/dev/null'
self.stdout_path = '/tmp/out.log'
self.stderr_path = '/tmp/err.log'
self.pidfile_path = '/var/run/img-storage-nas.pid'
self.pidfile_timeout = 5
self.function_dict = {
'map_zvol': self.map_zvol,
'unmap_zvol': self.unmap_zvol,
'zvol_mapped': self.zvol_mapped,
'zvol_unmapped': self.zvol_unmapped,
'list_zvols': self.list_zvols,
'del_zvol': self.del_zvol,
'get_zvol_attrs': self.get_zvol_attrs,
'set_zvol_attrs': self.set_zvol_attrs,
'get_attrs': self.get_attrs,
'set_attrs': self.set_attrs,
'del_attrs': self.del_attrs,
'zvol_synced': self.zvol_synced,
}
self.nc = NodeConfig.NodeConfig()
self.SQLITE_DB = '/opt/rocks/var/img_storage.db'
self.NODE_NAME = self.nc.NODE_NAME
self.ib_net = self.nc.SYNC_NETWORK
self.sync_result = None
self.results = {}
if self.nc.IMG_SYNC_WORKERS:
self.SYNC_WORKERS = int(self.nc.IMG_SYNC_WORKERS)
else:
self.SYNC_WORKERS = 5
self.SYNC_CHECK_TIMEOUT = 10
# try once/minute to add new jobs to the sync queue
self.SYNC_PULL_TIMEOUT = 60
# the default SYNC_PULL is 5 minutes
self.SYNC_PULL_DEFAULT = 60 * 5
self.ssl_options = self.nc.DATA.get("ssl_options", False)
if(self.ssl_options):
self.ssl_options = json.loads(self.ssl_options)
self.use_encryption = self.nc.DATA.get("use_encryption", False)
self.secur_server = self.nc.DATA.get("secur_server", False)
self.logger = \
logging.getLogger('imgstorage.imgstoragenas.NasDaemon')
self.ZVOLATTRS = ['frequency', 'nextsync',
'downloadspeed', 'uploadspeed']
def dbconnect(self):
""" connect to sqlite3 database, turn on foreign constraints """
con = sqlite3.connect(self.SQLITE_DB)
cur = con.cursor()
cur.execute('PRAGMA foreign_keys=ON')
return con
def getZvolAttr(self, zvol, attr=None):
""" Return a single named attribute for a zvol, or a dictionary of
attributes if attr=None """
rval = None
with self.dbconnect() as con:
cur = con.cursor()
if attr is not None:
cur.execute("SELECT %s FROM zvolattrs WHERE zvol='%s'" %
(attr, zvol))
row = cur.fetchone()
if row != None:
rval = row[0]
else:
cur.execute("SELECT * from zvolattrs WHERE zvol='%s'" % zvol)
row = cur.fetchone()
if row != None:
rval = dict((cur.description[i][0], value)
for (i, value) in enumerate(row))
return rval
def setZvolAttr(self, zvol, attr, value=None):
""" Set a single named attribute for a zvol. Set to Null of value is None """
with self.dbconnect() as con:
cur = con.cursor()
cur.execute(
'SELECT count(*) FROM zvolattrs WHERE zvol = ?', [zvol])
if cur.fetchone()[0] == 0:
cur.execute('INSERT INTO zvolattrs(zvol) VALUES(?)', [zvol])
if value is None:
setStmt = "SET %s=NULL" % attr
elif isinstance(value, int):
setStmt = "SET %s=%d" % (attr, value)
else:
setStmt = "SET %s='%s'" % (attr, value)
cur.execute(""" UPDATE zvolattrs %s WHERE zvol='%s'""" %
(setStmt, zvol))
con.commit()
def getAttr(self, attr=None):
""" Return a single named global attribute or a dictionary of
all attributes if attr=None. All values are type string """
rval = None
with self.dbconnect() as con:
cur = con.cursor()
if attr is not None:
cur.execute("SELECT value FROM globals WHERE attr='%s'" %
(attr))
row = cur.fetchone()
if row != None:
rval = row[0]
else:
cur.execute("SELECT attr,value FROM globals")
rval = {}
for row in cur.fetchall():
attr, value = row
rval[attr] = value
return rval
def setAttr(self, attr, value=None):
""" Set a single named attribute.Set to Null of value is None """
with self.dbconnect() as con:
if value is not None:
value = str(value)
cur = con.cursor()
cur.execute('''INSERT OR REPLACE INTO globals(attr,value)
VALUES(?,?)''', [attr, value] )
con.commit()
def deleteAttr(self, attr):
""" delete single named attribute """
with self.dbconnect() as con:
cur = con.cursor()
cur.execute('''DELETE FROM globals WHERE attr="%s"''' % attr )
con.commit()
# Attribute get/set messages
def get_zvol_attrs(self, message, props):
# input: get_zvol_attr messages
# output: get_zvol_attr message --> requestor
zvol = message['zvol']
try:
attrs = self.getZvolAttr(zvol)
attrs['zvol'] = zvol
except Exception as err:
self.failAction(props.reply_to, 'get_zvol_attrs', str(err))
return
reply = json.dumps({'action': 'get_zvol_attrs', 'status': 'success',
'body': attrs})
self.queue_connector.publish_message(reply, exchange='',
routing_key=props.reply_to)
def set_zvol_attrs(self, message, props):
# input: set_zvol_attr message
# output: set_zvol_attr message --> requestor
# state updates: zvols attr table
zvol = message['zvol']
try:
for k in message.keys():
if k in self.ZVOLATTRS:
self.setZvolAttr(zvol, k, message[k])
except Exception as err:
self.failAction(props.reply_to, 'set_zvol_attrs', str(err))
return
reply = json.dumps({'action': 'set_zvol_attrs', 'status': 'success'})
self.queue_connector.publish_message(reply, exchange='',
routing_key=props.reply_to)
def get_attrs(self, message, props):
# input: get_attrs messages
# output: get_attrs message --> requestor
try:
attrs = self.getAttr()
except Exception as err:
self.failAction(props.reply_to, 'get_attrs', str(err))
return
reply = json.dumps({'action': 'get_attrs', 'status': 'success',
'attrs': attrs})
self.queue_connector.publish_message(reply, exchange='',
routing_key=props.reply_to)
def set_attrs(self, message, props):
# input: set_attrs message
# output: set_attrs message --> requestor
# state updates: attrs table
try:
attrs = message['attrs']
for k in attrs.keys():
self.setAttr(k, attrs[k])
except Exception as err:
self.failAction(props.reply_to, 'set_attrs', str(err))
return
reply = json.dumps({'action': 'set_attrs', 'status': 'success'})
self.queue_connector.publish_message(reply, exchange='',
routing_key=props.reply_to)
def del_attrs(self, message, props):
# input: del_attrs message
# output: del_attrs message --> requestor
# state updates: attrs table
try:
for attr in message['attrs']:
self.deleteAttr(attr)
except Exception as err:
self.failAction(props.reply_to, 'del_attrs', str(err))
return
reply = json.dumps({'action': 'del_attrs', 'status': 'success'})
self.queue_connector.publish_message(reply, exchange='',
routing_key=props.reply_to)
############ Main Run Method ###################
def run(self):
self.pool = ThreadPool(processes=self.SYNC_WORKERS)
with self.dbconnect() as con:
cur = con.cursor()
cur.execute('''CREATE TABLE IF NOT EXISTS zvol_calls(
zvol TEXT PRIMARY KEY NOT NULL,
reply_to TEXT NOT NULL,
time INT NOT NULL)''')
cur.execute('''CREATE TABLE IF NOT EXISTS zvols(
zvol TEXT PRIMARY KEY NOT NULL,
zpool TEXT,
iscsi_target TEXT UNIQUE,
remotehost TEXT,
remotepool TEXT,
sync BOOLEAN)''')
cur.execute('''CREATE TABLE IF NOT EXISTS zvolattrs(
zvol TEXT PRIMARY KEY NOT NULL,
frequency INT DEFAULT NULL,
nextsync INT DEFAULT NULL,
downloadspeed INT DEFAULT NULL,
uploadspeed INT DEFAULT NULL)''')
cur.execute('''CREATE TABLE IF NOT EXISTS sync_queue(
zvol TEXT PRIMARY KEY NOT NULL,
zpool TEXT NOT NULL,
remotehost TEXT,
remotepool TEXT,
is_sending BOOLEAN,
is_delete_remote BOOLEAN,
time INT)''')
cur.execute('''CREATE TABLE IF NOT EXISTS globals(
attr TEXT PRIMARY KEY NOT NULL,
value TEXT)''')
cur.execute('DELETE FROM sync_queue')
con.commit()
# Record all parameters in the configuration file in globals
for key in self.nc.DATA.keys():
self.setAttr(key, self.nc.DATA[key])
if self.getAttr('frequency') is None:
self.setAttr('frequency', self.SYNC_PULL_DEFAULT)
self.queue_connector = RabbitMQCommonClient('rocks.vm-manage',
'direct', "img-storage", "img-storage",
self.process_message, lambda a:
self.startup(),
routing_key=self.nc.NODE_NAME,
ssl = True,
ssl_options = self.ssl_options,
encryption = self.use_encryption,
frontend = self.nc.FRONTEND_NAME,
secur_server = self.secur_server
)
self.queue_connector.run()
def failAction(
self,
routing_key,
action,
error_message,
):
if routing_key != None and action != None:
self.queue_connector.publish_message(json.dumps({'action': action,
'status': 'error', 'error': error_message}),
exchange='', routing_key=routing_key)
self.logger.error('Failed %s: %s' % (action, error_message))
def startup(self):
self.schedule_zvols_pull()
self.schedule_next_sync()
@coroutine
def map_zvol(self, message, props):
# input: map_zvol message
# output: map_zvol message --> remotehost
# state updates: zvol table
remotehost = message['remotehost']
remotepool = message['remotepool']
zpool_name = message['zpool']
zvol_name = message['zvol']
sync = message['sync']
try:
initiator = message['initiator']
except:
raise ActionError(
'Error when setting up: no initiator specified')
# print "XXX map_zvol (message): ", message
self.logger.debug('Setting zvol %s' % zvol_name)
with sqlite3.connect(self.SQLITE_DB) as con:
cur = con.cursor()
try:
self.lock_zvol(zvol_name, props.reply_to)
cur.execute(
'SELECT count(*) FROM zvols WHERE zvol = ?', [zvol_name])
volume = "%s/%s" % (zpool_name, zvol_name)
if cur.fetchone()[0] == 0:
""" Create a zvol, if it doesn't already exist """
# check if volume already exists
# print 'XXX checking if zvol %s exists' % volume
self.logger.debug('checking if zvol %s exists' % volume)
rcode = subprocess.call(["zfs", "list", volume])
# print 'XXX check complete (%s)' % volume
self.logger.debug('check complete (%s)' % volume)
if rcode != 0:
# create the zfs FS
yield runCommandBackground(zfs_create
+ ['-V', '%sgb' % message['size'], volume])
self.logger.debug('Created new zvol %s' % volume)
else:
self.logger.debug('Vol %s exists' % volume)
# Record the creation of the volume
cur.execute('''INSERT OR REPLACE INTO
zvols(zvol,zpool,iscsi_target,remotehost,remotepool,sync)
VALUES (?,?,?,?,?,?) '''
, (zvol_name, zpool_name, None, None, None, False))
con.commit()
# print 'XXX Created new zvol %s' % volume
self.logger.debug('Created new zvol %s' % volume)
cur.execute(
'SELECT remotehost FROM zvols WHERE zvol = ?', [zvol_name])
row = cur.fetchone()
if row != None and row[0] != None: # zvol is mapped
raise ActionError(
'Error when mapping zvol: already mapped')
ip = None
use_ib = False
if self.ib_net:
try:
ip = socket.gethostbyname(nodenameToDomain(remotehost, self.ib_net))
use_ib = True
except:
pass
if not use_ib:
try:
ip = socket.gethostbyname(remotehost)
except:
raise ActionError('Host %s is unknown'
% remotehost)
iscsi_target = ''
(out, err) = (yield runCommandBackground([
'/opt/rocks/bin/tgt-setup-lun-lock',
'-n',
zvol_name,
'-d',
'/dev/%s/%s' % (zpool_name, zvol_name),
initiator,
]))
for line in out:
if 'Creating new target' in line:
start = 'Creating new target (name='.__len__()
iscsi_target = line[start:line.index(',')]
for line in out:
if 'Created Target' in line:
iscsi_target = line.split()[-1]
# print 'XXX Mapped %s to iscsi target %s' % (zvol_name,
# iscsi_target)
self.logger.debug('Mapped %s to iscsi target %s'
% (zvol_name, iscsi_target))
# Update the Zvols table with the target, remote and sync
# attributes
cur.execute('''INSERT OR REPLACE INTO
zvols(zvol,zpool,iscsi_target,remotehost, remotepool,sync)
VALUES (?,?,?,?,?,?) '''
, (zvol_name, zpool_name, iscsi_target, remotehost, remotepool, sync))
con.commit()
# print 'XXX iscsi target %s inserted into DB' % iscsi_target
def failDeliver(
target,
zvol,
reply_to,
remotehost,
):
self.detach_target(target, True)
self.failAction(props.reply_to, 'zvol_mapped',
'Compute node %s is unavailable'
% remotehost)
self.release_zvol(zvol_name)
# Send a map_zvol message to remotehost
# XXX: Should rename this message
self.queue_connector.publish_message(json.dumps({
'action': 'map_zvol',
'target': iscsi_target,
'nas': (nodenameToDomain(self.NODE_NAME,
self.ib_net) if use_ib else self.NODE_NAME),
'size': message['size'],
'zvol': zvol_name,
'sync': sync,
'remotehost': remotehost,
'remotepool': remotepool,
}), remotehost, self.NODE_NAME, on_fail=lambda:
failDeliver(iscsi_target, zvol_name,
props.reply_to, remotehost))
self.logger.debug('Setting iscsi %s sent'
% iscsi_target)
except ActionError, err:
if not isinstance(err, ZvolBusyActionError):
self.release_zvol(zvol_name)
self.failAction(props.reply_to, 'zvol_mapped', str(err))
class NasDaemon():
def __init__(self):
self.stdin_path = '/dev/null'
self.stdout_path = '/tmp/out.log'
self.stderr_path = '/tmp/err.log'
self.pidfile_path = '/var/run/img-storage-nas.pid'
self.pidfile_timeout = 5
self.function_dict = {'set_zvol':self.set_zvol, 'tear_down':self.tear_down, 'zvol_attached':self.zvol_attached, 'zvol_detached': self.zvol_detached, 'list_zvols': self.list_zvols, 'del_zvol': self.del_zvol }
self.ZPOOL = 'tank'
self.SQLITE_DB = '/opt/rocks/var/img_storage.db'
self.NODE_NAME = RabbitMQLocator().NODE_NAME
self.logger = logging.getLogger(__name__)
db = rocks.db.helper.DatabaseHelper()
db.connect()
self.ib_net = db.getHostAttr(db.getHostname(), 'IB_net')
db.close()
def run(self):
with sqlite3.connect(self.SQLITE_DB) as con:
cur = con.cursor()
cur.execute('CREATE TABLE IF NOT EXISTS zvol_calls(zvol TEXT PRIMARY KEY NOT NULL, reply_to TEXT NOT NULL, time INT NOT NULL)')
cur.execute('CREATE TABLE IF NOT EXISTS zvols(zvol TEXT PRIMARY KEY NOT NULL, iscsi_target TEXT UNIQUE, hosting TEXT)')
con.commit()
self.queue_connector = RabbitMQCommonClient('rocks.vm-manage', 'direct', self.process_message)
self.queue_connector.run()
def failAction(self, routing_key, action, error_message):
if(routing_key != None and action != None):
self.queue_connector.publish_message({'action': action, 'status': 'error', 'error':error_message}, exchange='', routing_key=routing_key)
self.logger.error("Failed %s: %s"%(action, error_message))
"""
Received set_zvol command from frontend, passing to compute node
"""
def set_zvol(self, message, props):
hosting = message['hosting']
zvol_name = message['zvol']
self.logger.debug("Setting zvol %s"%zvol_name)
with sqlite3.connect(self.SQLITE_DB) as con:
cur = con.cursor()
try :
self.lock_zvol(zvol_name, props.reply_to)
cur.execute('SELECT count(*) FROM zvols WHERE zvol = ?',[zvol_name])
if(cur.fetchone()[0] == 0):
runCommand(['zfs', 'create', '-o', 'primarycache=metadata', '-o', 'volblocksize=128K', '-V', '%sgb'%message['size'], '%s/%s'%(self.ZPOOL, zvol_name)])
cur.execute('INSERT OR REPLACE INTO zvols VALUES (?,?,?) ',(zvol_name, None, None))
con.commit()
self.logger.debug('Created new zvol %s'%zvol_name)
cur.execute('SELECT iscsi_target FROM zvols WHERE zvol = ?',[zvol_name])
row = cur.fetchone()
if (row != None and row[0] != None): #zvol is mapped
raise ActionError('Error when mapping zvol: already mapped')
ip = None
use_ib = False
if(self.ib_net):
try:
ip = socket.gethostbyname('%s.%s'%(hosting, self.ib_net))
use_ib = True
except:
pass
if not use_ib:
try:
ip = socket.gethostbyname(hosting)
except:
raise ActionError('Host %s is unknown'%hosting)
iscsi_target = ''
out = runCommand(['tgt-setup-lun', '-n', zvol_name, '-d', '/dev/%s/%s'%(self.ZPOOL, zvol_name), ip])
for line in out:
if "Creating new target" in line:
iscsi_target = line['Creating new target (name='.__len__():line.index(',')]
self.logger.debug('Mapped %s to iscsi target %s'%(zvol_name, iscsi_target))
cur.execute('INSERT OR REPLACE INTO zvols VALUES (?,?,?) ',(zvol_name, iscsi_target,hosting))
con.commit()
def failDeliver(target, zvol, reply_to, hosting):
self.detach_target(target)
self.failAction(props.reply_to, 'zvol_attached', 'Compute node %s is unavailable'%hosting)
self.release_zvol(zvol_name)
self.queue_connector.publish_message(
{'action': 'set_zvol', 'target':iscsi_target, 'nas': ('%s.%s'%(self.NODE_NAME, self.ib_net)) if use_ib else self.NODE_NAME},
hosting,
self.NODE_NAME,
on_fail=lambda: failDeliver(iscsi_target, zvol_name, props.reply_to, hosting))
self.logger.debug("Setting iscsi %s sent"%iscsi_target)
except ActionError, err:
if not isinstance(err, ZvolBusyActionError): self.release_zvol(zvol_name)
self.failAction(props.reply_to, 'zvol_attached', str(err))
class NasDaemon:
def __init__(self):
## FIXME: user should be configurable at startup
self.imgUser = '******'
self.prefix = "IMG-STORAGE-"
self.stdin_path = '/dev/null'
self.stdout_path = '/tmp/out.log'
self.stderr_path = '/tmp/err.log'
self.pidfile_path = '/var/run/img-storage-nas.pid'
self.pidfile_timeout = 5
self.function_dict = {
'map_zvol': self.map_zvol,
'unmap_zvol': self.unmap_zvol,
'zvol_mapped': self.zvol_mapped,
'zvol_unmapped': self.zvol_unmapped,
'list_zvols': self.list_zvols,
'del_zvol': self.del_zvol,
'zvol_synced': self.zvol_synced,
}
self.SQLITE_DB = '/opt/rocks/var/img_storage.db'
self.NODE_NAME = NodeConfig.NODE_NAME
self.ib_net = NodeConfig.IB_NET
self.sync_result = None
self.results = {}
if NodeConfig.IMG_SYNC_WORKERS:
self.SYNC_WORKERS = int(NodeConfig.IMG_SYNC_WORKERS)
else:
self.SYNC_WORKERS = 5
self.SYNC_CHECK_TIMEOUT = 10
self.SYNC_PULL_TIMEOUT = 60 * 5
rocks.db.helper.DatabaseHelper().closeSession() # to reopen after daemonization
self.logger = \
logging.getLogger('imgstorage.imgstoragenas.NasDaemon')
def run(self):
self.pool = ThreadPool(processes=self.SYNC_WORKERS)
with sqlite3.connect(self.SQLITE_DB) as con:
cur = con.cursor()
cur.execute('''CREATE TABLE IF NOT EXISTS zvol_calls(
zvol TEXT PRIMARY KEY NOT NULL,
reply_to TEXT NOT NULL,
time INT NOT NULL)''')
cur.execute('''CREATE TABLE IF NOT EXISTS zvols(
zvol TEXT PRIMARY KEY NOT NULL,
zpool TEXT,
iscsi_target TEXT UNIQUE,
remotehost TEXT)''')
cur.execute('''CREATE TABLE IF NOT EXISTS sync_queue(
zvol TEXT PRIMARY KEY NOT NULL,
zpool TEXT NOT NULL,
remotehost TEXT,
is_sending BOOLEAN,
is_delete_remote BOOLEAN,
time INT)''')
con.commit()
self.queue_connector = RabbitMQCommonClient('rocks.vm-manage',
'direct', "img-storage", "img-storage",
self.process_message, lambda a: \
self.startup(),
routing_key=NodeConfig.NODE_NAME)
self.queue_connector.run()
def failAction(
self,
routing_key,
action,
error_message,
):
if routing_key != None and action != None:
self.queue_connector.publish_message(json.dumps({'action': action,
'status': 'error', 'error': error_message}),
exchange='', routing_key=routing_key)
self.logger.error('Failed %s: %s' % (action, error_message))
def startup(self):
self.schedule_zvols_pull()
self.schedule_next_sync()
@coroutine
def map_zvol(self, message, props):
remotehost = message['remotehost']
zpool_name = message['zpool']
zvol_name = message['zvol']
self.logger.debug('Setting zvol %s' % zvol_name)
with sqlite3.connect(self.SQLITE_DB) as con:
cur = con.cursor()
try:
self.lock_zvol(zvol_name, props.reply_to)
cur.execute('SELECT count(*) FROM zvols WHERE zvol = ?'
, [zvol_name])
volume = "%s/%s" % (zpool_name, zvol_name)
if cur.fetchone()[0] == 0:
""" Create a zvol, if it doesn't already exist """
# check if volume already exists
self.logger.debug('checking if zvol %s exists' % volume)
rcode = subprocess.call(["zfs","list",volume])
self.logger.debug('check complete (%s)' % volume)
if rcode != 0:
# create the zfs FS
yield runCommandBackground(zfs_create
+ ['-V', '%sgb' % message['size'], volume ])
self.logger.debug('Created new zvol %s' % volume)
else:
self.logger.debug('Vol %s exists' % volume)
cur.execute('INSERT OR REPLACE INTO zvols VALUES (?,?,?,?) '
, (zvol_name, None, None, None))
con.commit()
self.logger.debug('Created new zvol %s' % volume)
cur.execute('SELECT remotehost FROM zvols WHERE zvol = ?'
, [zvol_name])
row = cur.fetchone()
if row != None and row[0] != None: # zvol is mapped
raise ActionError('Error when mapping zvol: already mapped'
)
ip = None
use_ib = False
if self.ib_net:
try:
ip = socket.gethostbyname('%s.%s'
% (remotehost, self.ib_net))
use_ib = True
except:
pass
if not use_ib:
try:
ip = socket.gethostbyname(remotehost)
except:
raise ActionError('Host %s is unknown'
% remotehost)
iscsi_target = ''
(out, err) = (yield runCommandBackground([
'/opt/rocks/bin/tgt-setup-lun-lock',
'-n',
zvol_name,
'-d',
'/dev/%s/%s' % (zpool_name, zvol_name),
ip,
]))
for line in out:
if 'Creating new target' in line:
start = 'Creating new target (name='.__len__()
iscsi_target = line[start:line.index(',')]
self.logger.debug('Mapped %s to iscsi target %s'
% (zvol_name, iscsi_target))
cur.execute('INSERT OR REPLACE INTO zvols VALUES (?,?,?,?) '
, (zvol_name, zpool_name, iscsi_target,
remotehost))
con.commit()
def failDeliver(
target,
zvol,
reply_to,
remotehost,
):
self.detach_target(target, True)
self.failAction(props.reply_to, 'zvol_mapped',
'Compute node %s is unavailable'
% remotehost)
self.release_zvol(zvol_name)
self.queue_connector.publish_message(json.dumps({
'action': 'map_zvol',
'target': iscsi_target,
'nas': ('%s.%s' % (self.NODE_NAME,
self.ib_net) if use_ib else self.NODE_NAME),
'size': message['size'],
'zvol': zvol_name,
}), remotehost, self.NODE_NAME, on_fail=lambda : \
failDeliver(iscsi_target, zvol_name,
props.reply_to, remotehost))
self.logger.debug('Setting iscsi %s sent'
% iscsi_target)
except ActionError, err:
if not isinstance(err, ZvolBusyActionError):
self.release_zvol(zvol_name)
self.failAction(props.reply_to, 'zvol_mapped', str(err))
