def main():
parser = ArgumentParser(
description=("Utility for extracting and posting " \
"MythTV database logs."))
parser.add_argument('-a', '--application',
action='append',
dest='appname',
help=("Name of MythTV application to pull logs for. " \
"This argument can be called multiple times."))
parser.add_argument(
'-l',
'--list-applications',
action='store_true',
dest='showlist',
help="Show names of applications listed logged in database.")
parser.add_argument(
'-p',
'--profilename',
action='store',
dest='profile',
help=("Specify system profile to pull logs for. This is "
"typically the system hostname."))
parser.add_argument('-n', '--count',
action='store',
type=int,
dest='count',
default=1,
help=("Number of application instances prior to current " \
"to pull from the database."))
parser.add_argument('--engine',
action='store',
dest='engine',
default=Engine.getDefault(),
help="Pastebin site to upload to.")
parser.add_argument('--list-engines',
action='store_true',
dest='showengines',
help="Show names of supported Pastebin websites.")
res = parser.parse_args()
DB = DBCache()
if res.showlist:
print ' -- name -- -- instance count --'
for app in Log.getAppNames(res.profile, DB):
print '{0.name:<30} {0.count:^20}'.format(app)
sys.exit(0)
if res.showengines:
for engine in Engine.getEngines():
print '{0.name:<20}: {0.site}'.format(engine)
sys.exit(0)
if res.appname:
engine = Engine(res.engine)
for appname in res.appname:
for instance in Log.getLogs(appname, res.count, res.profile, DB):
engine.post(instance)
def __init__(self, backend=None, port=None, db=None):
if backend and port:
XMLConnection.__init__(self, backend, port)
self.db = db
return
self.db = DBCache(db)
self.log = MythLog('Python XML Connection')
if backend is None:
# use master backend
backend = self.db.settings.NULL.MasterServerIP
if re.match('(?:\d{1,3}\.){3}\d{1,3}',backend) or \
check_ipv6(backend):
# process ip address
host = self.db._gethostfromaddr(backend)
self.host = backend
self.port = int(self.db.settings[host].BackendStatusPort)
else:
# assume given a hostname
self.host = backend
self.port = int(self.db.settings[self.host].BackendStatusPort)
if not self.port:
# try a truncated hostname
self.host = backend.split('.')[0]
self.port = int(self.db.setting[self.host].BackendStatusPort)
if not self.port:
raise MythDBError(MythError.DB_SETTING,
backend + ': BackendStatusPort')
def getLogs(cls, appname, count, hostname=None, db=None):
db = DBCache(db)
if hostname is None:
hostname = db.gethostname()
with db as cursor:
cursor.execute(
"""SELECT DISTINCT pid
FROM logging
WHERE application=?
AND host=?
ORDER BY id DESC;""", (appname, hostname))
try:
instances = zip(*cursor.fetchall())[0]
except IndexError:
print "No logs found on system profile."
sys.exit(1)
if count == -1:
count = len(instances)
else:
count = min(count, len(instances))
for pid in instances[0:count]:
yield list(
cls._fromQuery(
"""WHERE application=?
AND host=?
AND pid=?
ORDER BY id;""",
(appname, hostname, pid)))
def __init__(self, backend=None, port=None, db=None):
if backend and port:
XMLConnection.__init__(self, backend, port)
self.db = db
return
self.db = DBCache(db)
self.log = MythLog('Python XML Connection')
if backend is None:
# use master backend
backend = self.db.getMasterBackend()
# assume hostname from settings
host = self.db._getpreferredaddr(backend)
if host:
port = int(self.db.settings[backend].BackendStatusPort)
else:
# assume ip address
hostname = self.db._gethostfromaddr(backend)
host = backend
port = int(self.db.settings[hostname].BackendStatusPort)
# resolve ip address from name
reshost, resport = resolve_ip(host, port)
if not reshost:
raise MythDBError(MythError.DB_SETTING,
backend + ': BackendServerAddr')
if not resport:
raise MythDBError(MythError.DB_SETTING,
backend + ': BackendStatusPort')
self.host = host
self.port = port
def getAppNames(cls, hostname=None, db=None):
app = namedtuple('Application', ['name', 'count'])
db = DBCache(db)
if hostname is None:
hostname = db.gethostname()
with db as cursor:
cursor.execute(
"""SELECT application, count(1)
FROM (SELECT DISTINCT application, pid
FROM logging
WHERE host=?) AS `pids`
GROUP BY application;""", (hostname, ))
for row in cursor:
yield app(*row)
def findfile(filename, sgroup, db=None):
"""
findfile(filename, sgroup, db=None) -> StorageGroup object
Will search through all matching storage groups, searching for file.
Returns matching storage group upon success.
"""
db = DBCache(db)
for sg in db.getStorageGroup(groupname=sgroup):
# search given group
if sg.local:
if os.access(sg.dirname + filename, os.F_OK):
return sg
for sg in db.getStorageGroup():
# not found, search all other groups
if sg.local:
if os.access(sg.dirname + filename, os.F_OK):
return sg
return None
def __init__(self,
backend=None,
blockshutdown=False,
events=False,
db=None):
self.db = DBCache(db)
self.log = MythLog(self.logmodule, db=self.db)
self.hostname = None
self.sendcommands = True
self.blockshutdown = blockshutdown
self.receiveevents = events
if backend is None:
# no backend given, use master
self.host = self.db.settings.NULL.MasterServerIP
self.hostname = self.db._gethostfromaddr(self.host)
else:
backend = backend.strip('[]')
query = "SELECT hostname FROM settings WHERE value=? AND data=?"
if self._reip.match(backend):
# given backend is IP address
self.host = backend
self.hostname = self.db._gethostfromaddr(
backend, 'BackendServerIP')
elif check_ipv6(backend):
# given backend is IPv6 address
self.host = backend
self.hostname = self.db._gethostfromaddr(
backend, 'BackendServerIP6')
else:
# given backend is hostname, pull address from database
self.hostname = backend
self.host = self.db._getpreferredaddr(backend)
# lookup port from database
self.port = int(self.db.settings[self.hostname].BackendServerPort)
if not self.port:
raise MythDBError(MythError.DB_SETTING, 'BackendServerPort',
self.port)
self._ident = '%s:%d' % (self.host, self.port)
if self._ident in self._shared:
# existing connection found
self._conn = self._shared[self._ident]
if self.sendcommands:
if self._conn.command is None:
self._conn.command = self._newcmdconn()
elif self.blockshutdown:
# upref block of shutdown
# issue command to backend if needed
self._conn.blockshutdown += 1
if self._conn.blockshutdown == 1:
self._conn.command.blockShutdown()
if self.receiveevents:
if self._conn.event is None:
self._conn.event = self._neweventconn()
else:
# no existing connection, create new
self._conn = self._ConnHolder()
if self.sendcommands:
self._conn.command = self._newcmdconn()
if self.blockshutdown:
self._conn.blockshutdown = 1
if self.receiveevents:
self._conn.event = self._neweventconn()
self._shared[self._ident] = self._conn
self._events = self._listhandlers()
for func in self._events:
self.registerevent(func)
def ftopen(file, mode, forceremote=False, nooverwrite=False, db=None, \
chanid=None, starttime=None, download=False):
"""
ftopen(file, mode, forceremote=False, nooverwrite=False, db=None)
-> FileTransfer object
-> file object
Method will attempt to open file locally, falling back to remote access
over mythprotocol if necessary.
'forceremote' will force a FileTransfer object if possible.
'file' takes a standard MythURI:
myth://<group>@<host>:<port>/<path>
'mode' takes a 'r' or 'w'
'nooverwrite' will refuse to open a file writable,
if a local file is found.
"""
db = DBCache(db)
log = MythLog('Python File Transfer', db=db)
reuri = re.compile(\
'myth://((?P<group>.*)@)?(?P<host>[\[\]a-zA-Z0-9_\-\.]*)(:[0-9]*)?/(?P<file>.*)')
reip = re.compile('(?:\d{1,3}\.){3}\d{1,3}')
if mode not in ('r', 'w'):
raise TypeError("File I/O must be of type 'r' or 'w'")
if chanid and starttime:
protoopen = lambda host, file, storagegroup: \
RecordFileTransfer(host, file, storagegroup,\
mode, chanid, starttime, db)
elif download:
protoopen = lambda host, lfile, storagegroup: \
DownloadFileTransfer(host, lfile, storagegroup, \
mode, file, db)
else:
protoopen = lambda host, file, storagegroup: \
FileTransfer(host, file, storagegroup, mode, db)
# process URI (myth://<group>@<host>[:<port>]/<path/to/file>)
match = reuri.match(file)
if match is None:
raise MythError('Invalid FileTransfer input string: ' + file)
host = match.group('host')
filename = match.group('file')
sgroup = match.group('group')
if sgroup is None:
sgroup = 'Default'
# get full system name
host = host.strip('[]')
if reip.match(host) or check_ipv6(host):
host = db._gethostfromaddr(host)
# user forced to remote access
if forceremote:
if (mode == 'w') and (filename.find('/') != -1):
raise MythFileError(MythError.FILE_FAILED_WRITE, file,
'attempting remote write outside base path')
if nooverwrite and FileOps(host, db=db).fileExists(filename, sgroup):
raise MythFileError(MythError.FILE_FAILED_WRITE, file,
'refusing to overwrite existing file')
return protoopen(host, filename, sgroup)
if mode == 'w':
# check for pre-existing file
path = FileOps(host, db=db).fileExists(filename, sgroup)
sgs = list(db.getStorageGroup(groupname=sgroup))
if path is not None:
if nooverwrite:
raise MythFileError(MythError.FILE_FAILED_WRITE, file,
'refusing to overwrite existing file')
for sg in sgs:
if sg.dirname in path:
if sg.local:
return open(sg.dirname + filename, mode)
else:
return protoopen(host, filename, sgroup)
# prefer local storage for new files
for i, v in reversed(list(enumerate(sgs))):
if not v.local:
sgs.pop(i)
else:
st = os.statvfs(v.dirname)
v.free = st[0] * st[3]
if len(sgs) > 0:
# choose path with most free space
sg = sorted(sgs, key=lambda sg: sg.free, reverse=True)[0]
# create folder if it does not exist
if filename.find('/') != -1:
path = sg.dirname + filename.rsplit('/', 1)[0]
if not os.access(path, os.F_OK):
os.makedirs(path)
log(log.FILE, log.INFO, 'Opening local file (w)',
sg.dirname + filename)
return open(sg.dirname + filename, mode)
# fallback to remote write
else:
if filename.find('/') != -1:
raise MythFileError(
MythError.FILE_FAILED_WRITE, file,
'attempting remote write outside base path')
return protoopen(host, filename, sgroup)
else:
# search for file in local directories
sg = findfile(filename, sgroup, db)
if sg is not None:
# file found, open local
log(log.FILE, log.INFO, 'Opening local file (r)',
sg.dirname + filename)
return open(sg.dirname + filename, mode)
else:
# file not found, open remote
return protoopen(host, filename, sgroup)
def __init__(self, backend=None, blockshutdown=False, events=False, db=None):
self.db = DBCache(db)
self.log = MythLog(self.logmodule, db=self.db)
self.hostname = None
self.sendcommands = True
self.blockshutdown = blockshutdown
self.receiveevents = events
if backend is None:
# use master backend
backend = self.db.getMasterBackend()
else:
backend = backend.strip('[]')
# assume backend is hostname from settings
host = self.db._getpreferredaddr(backend)
if host:
port = int(self.db.settings[backend].BackendServerPort)
self.hostname = backend
else:
# assume ip address
self.hostname = self.db._gethostfromaddr(backend)
host = backend
port = int(self.db.settings[self.hostname].BackendServerPort)
# resolve ip address from name
reshost, resport = resolve_ip(host,port)
if not reshost:
raise MythDBError(MythError.DB_SETTING,
backend+': BackendServerAddr')
if not resport:
raise MythDBError(MythError.DB_SETTING,
backend+': BackendServerPort')
self.host = host
self.port = port
self._ident = '%s:%d' % (self.host, self.port)
if self._ident in self._shared:
# existing connection found
self._conn = self._shared[self._ident]
if self.sendcommands:
if self._conn.command is None:
self._conn.command = self._newcmdconn()
elif self.blockshutdown:
# upref block of shutdown
# issue command to backend if needed
self._conn.blockshutdown += 1
if self._conn.blockshutdown == 1:
self._conn.command.blockShutdown()
if self.receiveevents:
if self._conn.event is None:
self._conn.event = self._neweventconn()
else:
# no existing connection, create new
self._conn = self._ConnHolder()
if self.sendcommands:
self._conn.command = self._newcmdconn()
if self.blockshutdown:
self._conn.blockshutdown = 1
if self.receiveevents:
self._conn.event = self._neweventconn()
self._shared[self._ident] = self._conn
self._events = self._listhandlers()
for func in self._events:
self.registerevent(func)
