def __init__(self, backend=None, noshutdown=False, db=None, opts=None):
self.db = DBCache(db)
self.log = MythLog(self.logmodule, db=self.db)
self.hostname = None
if opts is None:
self.opts = BEConnection.BEConnOpts(noshutdown)
else:
self.opts = opts
if backend is None:
# no backend given, use master
self.host = self.db.settings.NULL.MasterServerIP
else:
if self._reip.match(backend):
# given backend is IP address
self.host = backend
else:
# given backend is hostname, pull address from database
self.hostname = backend
self.host = self.db.settings[backend].BackendServerIP
if not self.host:
raise MythDBError(MythError.DB_SETTING, 'BackendServerIP',
backend)
if self.hostname is None:
# reverse lookup hostname from address
with self.db.cursor(self.log) as cursor:
if cursor.execute(
"""SELECT hostname FROM settings
WHERE value='BackendServerIP'
AND data=?""", [self.host]) == 0:
# no match found
raise MythDBError(MythError.DB_SETTING, 'BackendServerIP',
self.host)
self.hostname = cursor.fetchone()[0]
# lookup port from database
self.port = int(self.db.settings[self.hostname].BackendServerPort)
if not self.port:
raise MythDBError(MythError.DB_SETTING, 'BackendServerPort',
self.port)
self._uuid = uuid4()
self._ident = '%s:%d' % (self.host, self.port)
if self._ident in self._shared:
# existing connection found
# register and reconnect if necessary
self.be = self._shared[self._ident]
self.be.registeruser(self._uuid, self.opts)
self.be.reconnect()
else:
# no existing connection, create new
self.be = BEConnection(self.host, self.port, \
self.db.gethostname(), self.opts)
self.be.registeruser(self._uuid, self.opts)
self._shared[self._ident] = self.be
def __init__(self, path=None, setting=None, db=None, useshell=True):
DBCache.__init__(self, db=db)
self.log = MythLog(self.logmodule, db=self)
self.path = None
if setting is not None:
# pull setting from database, substitute from argument if needed
host = self.gethostname()
self.path = self.settings[host][setting]
if (self.path is None) and (path is None):
raise MythDBError(MythError.DB_SETTING, setting, host)
if self.path is None:
# setting not given, use path from argument
if path is None:
raise MythError('Invalid input to System()')
self.path = path
cmd = self.path.split()[0]
if self.path.startswith('/'):
# test full given path
if not os.access(cmd, os.F_OK):
raise MythFileError('Defined executable path does not exist.')
else:
# search command from PATH
for folder in os.environ['PATH'].split(':'):
if os.access(os.path.join(folder, cmd), os.F_OK):
self.path = os.path.join(folder, self.path)
break
else:
raise MythFileError('Defined executable path does not exist.')
self.returncode = 0
self.stderr = ''
self.useshell = useshell
def __init__(self, parent, field_name, editable=True):
self._parent = weakref.proxy(parent)
self._field = field_name
field = parent._db.tablefields[parent._table][self._field].type
if field[:4] != 'set(':
raise MythDBError("ParseSet error. "+\
"Field '%s' not of type 'set()'" % self._field)
self._enum = dict([(t,2**i) for i,t in enumerate([type.strip("'")\
for type in field[4:-1].split(',')])])
self._static = not editable
def run(self):
if self._schema_name is None:
raise MythDBError('Schema update failed, variable name not set')
if self.db.settings.NULL[self._schema_name] is None:
self.db.settings.NULL[self._schema_name] = self.create()
origschema = int(self.db.settings.NULL[self._schema_name])
schema = origschema
try:
while True:
newschema = eval('self.up%d()' % schema)
self.log(MythLog.DATABASE,
'successfully updated from %d to %d' %\
(schema, newschema))
schema = newschema
self.db.settings.NULL[self._schema_name] = schema
except AttributeError, e:
self.log(MythLog.DATABASE|MythLog.IMPORTANT,
'failed at %d' % schema, 'no handler method')
raise MythDBError('Schema update failed, '
"SchemaUpdate has no function 'up%s'" % schema)
def __init__(self, table=None, tableto=None, fields=None, \
fieldsto=None, fieldsfrom=None):
if (table is None) or (tableto is None) or \
((fields is None) and \
((fieldsto is None) or (fieldsfrom is None))):
raise MythDBError('Invalid input to databaseSearch.Join.')
self.table = table
self.tableto = tableto
if fields:
self.fieldsfrom = fields
self.fieldsto = fields
else:
self.fieldsfrom = fieldsfrom
self.fieldsto = fieldsto
def execute(self, query, args=None):
"""
Execute a query.
query -- string, query to execute on server
args -- optional sequence, parameters to use with query.
Returns long integer rows affected, if any
"""
query = self._sanitize(query)
self.log_query(query, args)
try:
if args:
return super(LoggedCursor, self).execute(query, args)
return super(LoggedCursor, self).execute(query)
except Exception, e:
raise MythDBError(MythDBError.DB_RAW, e.args)
def executemany(self, query, args):
"""
Execute a multi-row query.
query -- string, query to execute on server
args -- Sequence of sequences, parameters to use with query.
Returns long integer rows affected, if any.
This method improves performance on multiple-row INSERT and
REPLACE. Otherwise it is equivalent to looping over args with
execute().
"""
query = self._sanitize(query)
self.log_query(query, args)
try:
return super(LoggedCursor, self).executemany(query, args)
except Exception, e:
raise MythDBError(MythDBError.DB_RAW, e.args)
def execute(self, query, args=None):
"""
Execute a query.
query -- string, query to execute on server
args -- optional sequence or mapping, parameters to use with query.
Note: If args is a sequence, then %s must be used as the
parameter placeholder in the query. If a mapping is used,
%(key)s must be used as the placeholder.
Returns long integer rows affected, if any
"""
self.ping()
query = self._sanitize(query)
self.log_query(query, args)
try:
if args is None:
return super(LoggedCursor, self).execute(query)
return super(LoggedCursor, self).execute(query, args)
except Exception, e:
raise MythDBError(MythDBError.DB_RAW, e.args)
def create(self):
raise MythDBError('Schema creation failed, method not defined.')
self.db.settings.NULL[self._schema_name] = schema
except AttributeError, e:
self.log(MythLog.GENERAL, MythLog.CRIT, 'failed at %d' % schema,
'no handler method')
raise MythDBError('Schema update failed, '
"SchemaUpdate has no function 'up%s'" % schema)
except StopIteration:
if schema != origschema:
self.log(MythLog.GENERAL, MythLog.NOTICE,
'%s update complete' % self._schema_name)
pass
except Exception, e:
raise MythDBError(MythError.DB_SCHEMAUPDATE, e.args)
def create(self):
raise MythDBError('Schema creation failed, method not defined.')
class databaseSearch(object):
# decorator class for database searches
"""
Decorator class
Decorated functions must accept three keywords: init, key, value.
'init' will be set once to True, to initialize the decorator. During
initialization, the decorated class must return a tuple
of the following format
(<table name>, -- Primary table to pull data from.
def __init__(self,
backend=None,
blockshutdown=False,
events=False,
db=None):
self.db = DBCache(db)
self.log = MythLog(self.logmodule, db=self.db)
self.hostname = None
self.sendcommands = True
self.blockshutdown = blockshutdown
self.receiveevents = events
if backend is None:
# no backend given, use master
self.host = self.db.settings.NULL.MasterServerIP
else:
backend = backend.strip('[]')
if self._reip.match(backend):
# given backend is IP address
self.host = backend
elif check_ipv6(backend):
# given backend is IPv6 address
self.host = backend
else:
# given backend is hostname, pull address from database
self.hostname = backend
self.host = self.db.settings[backend].BackendServerIP
if not self.host:
raise MythDBError(MythError.DB_SETTING, 'BackendServerIP',
backend)
if self.hostname is None:
# reverse lookup hostname from address
with self.db.cursor(self.log) as cursor:
if cursor.execute(
"""SELECT hostname FROM settings
WHERE value='BackendServerIP'
AND data=?""", [self.host]) == 0:
# no match found
raise MythDBError(MythError.DB_SETTING, 'BackendServerIP',
self.host)
self.hostname = cursor.fetchone()[0]
# lookup port from database
self.port = int(self.db.settings[self.hostname].BackendServerPort)
if not self.port:
raise MythDBError(MythError.DB_SETTING, 'BackendServerPort',
self.port)
self._ident = '%s:%d' % (self.host, self.port)
if self._ident in self._shared:
# existing connection found
self._conn = self._shared[self._ident]
if self.sendcommands:
if self._conn.command is None:
self._conn.command = self._newcmdconn()
elif self.blockshutdown:
# upref block of shutdown
# issue command to backend if needed
self._conn.blockshutdown += 1
if self._conn.blockshutdown == 1:
self._conn.command.blockShutdown()
if self.receiveevents:
if self._conn.event is None:
self._conn.event = self._neweventconn()
else:
# no existing connection, create new
self._conn = self._ConnHolder()
if self.sendcommands:
self._conn.command = self._newcmdconn()
if self.blockshutdown:
self._conn.blockshutdown = 1
if self.receiveevents:
self._conn.event = self._neweventconn()
self._shared[self._ident] = self._conn
self._events = self._listhandlers()
for func in self._events:
self.registerevent(func)
def ftopen(file, mode, forceremote=False, nooverwrite=False, db=None, \
chanid=None, starttime=None, download=False):
"""
ftopen(file, mode, forceremote=False, nooverwrite=False, db=None)
-> FileTransfer object
-> file object
Method will attempt to open file locally, falling back to remote access
over mythprotocol if necessary.
'forceremote' will force a FileTransfer object if possible.
'file' takes a standard MythURI:
myth://<group>@<host>:<port>/<path>
'mode' takes a 'r' or 'w'
'nooverwrite' will refuse to open a file writable,
if a local file is found.
"""
db = DBCache(db)
log = MythLog('Python File Transfer', db=db)
reuri = re.compile(\
'myth://((?P<group>.*)@)?(?P<host>[\[\]a-zA-Z0-9_\-\.]*)(:[0-9]*)?/(?P<file>.*)')
reip = re.compile('(?:\d{1,3}\.){3}\d{1,3}')
if mode not in ('r', 'w'):
raise TypeError("File I/O must be of type 'r' or 'w'")
if chanid and starttime:
protoopen = lambda host, file, storagegroup: \
RecordFileTransfer(host, file, storagegroup,\
mode, chanid, starttime, db)
elif download:
protoopen = lambda host, lfile, storagegroup: \
DownloadFileTransfer(host, lfile, storagegroup, \
mode, file, db)
else:
protoopen = lambda host, file, storagegroup: \
FileTransfer(host, file, storagegroup, mode, db)
# process URI (myth://<group>@<host>[:<port>]/<path/to/file>)
match = reuri.match(file)
if match is None:
raise MythError('Invalid FileTransfer input string: ' + file)
host = match.group('host')
filename = match.group('file')
sgroup = match.group('group')
if sgroup is None:
sgroup = 'Default'
# get full system name
host = host.strip('[]')
if reip.match(host) or check_ipv6(host):
with db.cursor(log) as cursor:
if cursor.execute(
"""SELECT hostname FROM settings
WHERE value='BackendServerIP'
AND data=%s""", host) == 0:
raise MythDBError(MythError.DB_SETTING, \
'BackendServerIP', backend)
host = cursor.fetchone()[0]
# user forced to remote access
if forceremote:
if (mode == 'w') and (filename.find('/') != -1):
raise MythFileError(MythError.FILE_FAILED_WRITE, file,
'attempting remote write outside base path')
if nooverwrite and FileOps(host, db=db).fileExists(filename, sgroup):
raise MythFileError(MythError.FILE_FAILED_WRITE, file,
'refusing to overwrite existing file')
return protoopen(host, filename, sgroup)
if mode == 'w':
# check for pre-existing file
path = FileOps(host, db=db).fileExists(filename, sgroup)
sgs = list(db.getStorageGroup(groupname=sgroup))
if path is not None:
if nooverwrite:
raise MythFileError(MythError.FILE_FAILED_WRITE, file,
'refusing to overwrite existing file')
for sg in sgs:
if sg.dirname in path:
if sg.local:
return open(sg.dirname + filename, mode)
else:
return protoopen(host, filename, sgroup)
# prefer local storage for new files
for i, v in reversed(list(enumerate(sgs))):
if not v.local:
sgs.pop(i)
else:
st = os.statvfs(v.dirname)
v.free = st[0] * st[3]
if len(sgs) > 0:
# choose path with most free space
sg = sorted(sgs, key=lambda sg: sg.free, reverse=True)[0]
# create folder if it does not exist
if filename.find('/') != -1:
path = sg.dirname + filename.rsplit('/', 1)[0]
if not os.access(path, os.F_OK):
os.makedirs(path)
log(log.FILE, log.INFO, 'Opening local file (w)',
sg.dirname + filename)
return open(sg.dirname + filename, mode)
# fallback to remote write
else:
if filename.find('/') != -1:
raise MythFileError(
MythError.FILE_FAILED_WRITE, file,
'attempting remote write outside base path')
return protoopen(host, filename, sgroup)
else:
# search for file in local directories
sg = findfile(filename, sgroup, db)
if sg is not None:
# file found, open local
log(log.FILE, log.INFO, 'Opening local file (r)',
sg.dirname + filename)
return open(sg.dirname + filename, mode)
else:
# file not found, open remote
return protoopen(host, filename, sgroup)