def dlexpect(self, pattern, flags=0, deadline=None):
"""Loop recv listening for a provided regular expression."""
# pull default timeout
if deadline is None:
deadline = self._deadline
if deadline < 1000:
deadline += time()
buff = StringIO()
# loop until pattern has been found
while not pattern.search(buff.getvalue()):
# wait for data on the socket
t = time()
timeout = (deadline - t) if (deadline - t > 0) else 0.0
if len(select([self], [], [], timeout)[0]) == 0:
# deadline reached, terminate
return ''
# append response to buffer
p = buff.tell()
try:
buff.write(self.recv(100, flags))
except socket.error, e:
raise MythError(MythError.SOCKET, e.args)
if buff.tell() == p:
# no data read from a 'ready' socket, connection terminated
raise MythError(MythError.CLOSEDSOCKET)
if timeout == 0:
break
def _setsyslog(cls, facility=LOGFACILITY.USER):
cls._initlogger()
try:
facility = int(facility)
for fac in dir(LOGFACILITY):
if '_' in fac:
continue
if getattr(LOGFACILITY, fac) == facility:
facility = 'LOG_' + fac
break
else:
raise MythError("Invalid syslog facility")
except ValueError:
if not facility.startswith('LOG_'):
facility = 'LOG_' + facility.upper()
if not hasattr(LOGFACILITY, facility[4:]):
raise MythError("Invalid syslog facility")
cls._SYSLOG = facility
syslog.openlog(argv[0].rsplit('/', 1)[1],
syslog.LOG_NDELAY | syslog.LOG_PID,
getattr(syslog, facility))
cls._logwrite = cls._logsyslog
if cls._LOGFILE:
if cls._LOGFILE.fileno() != 1:
cls._LOGFILE.close()
cls._LOGFILE = None
def dlrecv(self, bufsize, flags=0, deadline=None):
# pull default timeout
if deadline is None:
deadline = self._deadline
if deadline < 1000:
deadline += time()
buff = StringIO()
# loop until necessary data has been received
while bufsize > buff.tell():
# wait for data on the socket
t = time()
timeout = (deadline - t) if (deadline - t > 0) else 0.0
if len(select([self], [], [], timeout)[0]) == 0:
# deadline reached, terminate
return u''
# append response to buffer
p = buff.tell()
try:
buff.write(self.recv(bufsize - buff.tell(), flags))
except socket.error, e:
raise MythError(MythError.SOCKET, e.args)
if buff.tell() == p:
# no data read from a 'ready' socket, connection terminated
raise MythError(MythError.SOCKET,
(54, 'Connection reset by peer'))
if timeout == 0:
break
def __init__(self, path=None, setting=None, db=None, useshell=True):
DBCache.__init__(self, db=db)
self.log = MythLog(self.logmodule, db=self)
self.path = None
if setting is not None:
# pull setting from database, substitute from argument if needed
host = self.gethostname()
self.path = self.settings[host][setting]
if (self.path is None) and (path is None):
raise MythDBError(MythError.DB_SETTING, setting, host)
if self.path is None:
# setting not given, use path from argument
if path is None:
raise MythError('Invalid input to System()')
self.path = path
cmd = self.path.split()[0]
if self.path.startswith('/'):
# test full given path
if not os.access(cmd, os.F_OK):
raise MythFileError('Defined executable path does not exist.')
else:
# search command from PATH
for folder in os.environ['PATH'].split(':'):
if os.access(os.path.join(folder, cmd), os.F_OK):
self.path = os.path.join(folder, self.path)
break
else:
raise MythFileError('Defined executable path does not exist.')
self.returncode = 0
self.stderr = ''
self.useshell = useshell
def sendheader(self, data, flags=0):
"""Send data, prepending the length in the first 8 bytes."""
try:
self.log(MythLog.SOCKET|MythLog.NETWORK, MythLog.DEBUG, \
'write --> %d' % len(data), data)
data = '%-8d%s' % (len(data), data)
self.send(data, flags)
except socket.error, e:
raise MythError(MythError.SOCKET, e.args)
def __call__(self, *args, **kwargs):
if self.func is None:
if len(args) == 1:
self.func = args[0]
elif 'func' in kwargs:
self.func = kwargs['func']
if not callable(self.func):
raise MythError('_ProgramQuery must receive a callable '+\
'before it is functional')
self.__doc__ = self.func.__doc__
self.__name__ = self.func.__name__
self.__module__ = self.func.__module__
return self
elif self.inst is None:
raise MythError('Call to uninitialized _ProgramQuery instance')
if self.sorted:
return self.sortedrun(*args, **kwargs)
return self.run(*args, **kwargs)
def _runcmd(self, cmd):
self.log(MythLog.FILE, 'Running external command', cmd)
fd = Popen(cmd, stdout=-1, stderr=-1, shell=True)
self.returncode = fd.wait()
stdout,self.stderr = fd.communicate()
if self.returncode:
raise MythError(MythError.SYSTEM,self.returncode,cmd,self.stderr)
return stdout
def _runshared(self, fd, cmd):
pmap = {fd.stdout:'', fd.stderr:''}
while fd.poll() is None:
socks = select([fd.stdout, fd.stderr],[],[])
for sock in socks[0]:
pmap[sock] += sock.read()
self.stderr = pmap[fd.stderr]
self.returncode = fd.poll()
if self.returncode:
raise MythError(MythError.SYSTEM,self.returncode,cmd,self.stderr)
return pmap[fd.stdout]
def _process(self, data):
"""
Accepts a list of data, processes according to specified types,
and returns a dictionary
"""
if self._field_type != 'Pass':
if len(data) != len(self._field_type):
raise MythError('Incorrect raw input length to DictData()')
data = list(data)
for i, v in enumerate(data):
if v == '':
data[i] = None
else:
data[i] = self._trans[self._field_type[i]](v)
return dict(zip(self._field_order, data))
def __init__(self, func):
# set function and update strings
self.func = func
self.__doc__ = self.func.__doc__
self.__name__ = self.func.__name__
self.__module__ = self.func.__module__
# set defaults
self.table = None
self.handler = None
self.require = ()
self.joins = ()
# pull in properties
self.func(self, self)
# sanity check
if (self.table is None) or (self.handler is None):
raise MythError('Improperly configured databaseSearch class')
def __init__(self):
self.log = MythLog('Python M-Search')
port = 1900
addr = '239.255.255.250'
self.dest = (addr, port)
self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM,
socket.IPPROTO_UDP)
self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
listening = False
while listening == False:
try:
self.sock.bind(('', port))
self.addr = (addr, port)
listening = True
except socket.error, e:
if port < 1910:
port += 1
else:
raise MythError(MythError.SOCKET, e)
def ftopen(file, mode, forceremote=False, nooverwrite=False, db=None, \
chanid=None, starttime=None, download=False):
"""
ftopen(file, mode, forceremote=False, nooverwrite=False, db=None)
-> FileTransfer object
-> file object
Method will attempt to open file locally, falling back to remote access
over mythprotocol if necessary.
'forceremote' will force a FileTransfer object if possible.
'file' takes a standard MythURI:
myth://<group>@<host>:<port>/<path>
'mode' takes a 'r' or 'w'
'nooverwrite' will refuse to open a file writable,
if a local file is found.
"""
db = DBCache(db)
log = MythLog('Python File Transfer', db=db)
reuri = re.compile(\
'myth://((?P<group>.*)@)?(?P<host>[\[\]a-zA-Z0-9_\-\.]*)(:[0-9]*)?/(?P<file>.*)')
reip = re.compile('(?:\d{1,3}\.){3}\d{1,3}')
if mode not in ('r', 'w'):
raise TypeError("File I/O must be of type 'r' or 'w'")
if chanid and starttime:
protoopen = lambda host, file, storagegroup: \
RecordFileTransfer(host, file, storagegroup,\
mode, chanid, starttime, db)
elif download:
protoopen = lambda host, lfile, storagegroup: \
DownloadFileTransfer(host, lfile, storagegroup, \
mode, file, db)
else:
protoopen = lambda host, file, storagegroup: \
FileTransfer(host, file, storagegroup, mode, db)
# process URI (myth://<group>@<host>[:<port>]/<path/to/file>)
match = reuri.match(file)
if match is None:
raise MythError('Invalid FileTransfer input string: ' + file)
host = match.group('host')
filename = match.group('file')
sgroup = match.group('group')
if sgroup is None:
sgroup = 'Default'
# get full system name
host = host.strip('[]')
if reip.match(host) or check_ipv6(host):
with db.cursor(log) as cursor:
if cursor.execute(
"""SELECT hostname FROM settings
WHERE value='BackendServerIP'
AND data=%s""", host) == 0:
raise MythDBError(MythError.DB_SETTING, \
'BackendServerIP', backend)
host = cursor.fetchone()[0]
# user forced to remote access
if forceremote:
if (mode == 'w') and (filename.find('/') != -1):
raise MythFileError(MythError.FILE_FAILED_WRITE, file,
'attempting remote write outside base path')
if nooverwrite and FileOps(host, db=db).fileExists(filename, sgroup):
raise MythFileError(MythError.FILE_FAILED_WRITE, file,
'refusing to overwrite existing file')
return protoopen(host, filename, sgroup)
if mode == 'w':
# check for pre-existing file
path = FileOps(host, db=db).fileExists(filename, sgroup)
sgs = list(db.getStorageGroup(groupname=sgroup))
if path is not None:
if nooverwrite:
raise MythFileError(MythError.FILE_FAILED_WRITE, file,
'refusing to overwrite existing file')
for sg in sgs:
if sg.dirname in path:
if sg.local:
return open(sg.dirname + filename, mode)
else:
return protoopen(host, filename, sgroup)
# prefer local storage for new files
for i, v in reversed(list(enumerate(sgs))):
if not v.local:
sgs.pop(i)
else:
st = os.statvfs(v.dirname)
v.free = st[0] * st[3]
if len(sgs) > 0:
# choose path with most free space
sg = sorted(sgs, key=lambda sg: sg.free, reverse=True)[0]
# create folder if it does not exist
if filename.find('/') != -1:
path = sg.dirname + filename.rsplit('/', 1)[0]
if not os.access(path, os.F_OK):
os.makedirs(path)
log(log.FILE, log.INFO, 'Opening local file (w)',
sg.dirname + filename)
return open(sg.dirname + filename, mode)
# fallback to remote write
else:
if filename.find('/') != -1:
raise MythFileError(
MythError.FILE_FAILED_WRITE, file,
'attempting remote write outside base path')
return protoopen(host, filename, sgroup)
else:
# search for file in local directories
sg = findfile(filename, sgroup, db)
if sg is not None:
# file found, open local
log(log.FILE, log.INFO, 'Opening local file (r)',
sg.dirname + filename)
return open(sg.dirname + filename, mode)
else:
# file not found, open remote
return protoopen(host, filename, sgroup)