def __init__(self,
path=None,
setting=None,
db=None,
useshell=True,
prefix=''):
DBCache.__init__(self, db=db)
self.log = MythLog(self.logmodule, db=self)
self.path = None
if setting is not None:
# pull local setting from database
host = self.gethostname()
self.path = self.settings[host][setting]
if self.path is None:
# see if that setting is applied globally
self.path = self.settings['NULL'][setting]
if self.path is None:
# not set globally either, use supplied default
self.path = path
if self.path is None:
# no default supplied, just error out
raise MythDBError(MythError.DB_SETTING, setting, host)
if self.path is None:
# setting not given, use path from argument
if path is None:
raise MythError('Invalid input to System()')
self.path = path
if prefix:
self.path = os.path.join(prefix, self.path)
cmd = self.path.split()[0]
if self.path.startswith('/'):
# test full given path
if not os.access(cmd, os.F_OK):
raise MythFileError('Defined executable path does not exist.')
else:
# search command from PATH
for folder in os.environ['PATH'].split(':'):
if os.access(os.path.join(folder, cmd), os.F_OK):
self.path = os.path.join(folder, self.path)
break
else:
raise MythFileError('Defined executable path does not exist.')
self.returncode = 0
self.stderr = ''
self.useshell = useshell
def write(self, data):
"""
FileTransfer.write(data) -> None
Requests over 128KB will be buffered internally
"""
if self.mode != 'w':
raise MythFileError('attempting to write to a read-only socket')
while len(data) > 0:
size = len(data)
# check size for buffering
if size > self._tsize:
size = self._tsize
buff = data[:size]
data = data[size:]
else:
buff = data
data = ''
# push data to server
self._pos += int(self.ftsock.send(buff))
if self._pos > self._size:
self._size = self._pos
# inform server of new data
self.backendCommand('QUERY_FILETRANSFER '\
+BACKEND_SEP.join(\
[str(self._sockno),
'WRITE_BLOCK',
str(size)]))
return
def read(self, size):
"""
FileTransfer.read(size) -> string of <size> characters
Requests over 128KB will be buffered internally.
"""
# some sanity checking
if self.mode != 'r':
raise MythFileError('attempting to read from a write-only socket')
if size == 0:
return ''
if self._pos + size > self._size:
size = self._size - self._pos
buff = ''
while len(buff) < size:
ct = size - len(buff)
if ct > self._tsize:
# drop size and bump counter if over limit
self._count += 1
ct = self._tsize
# request transfer
res = self.backendCommand('QUERY_FILETRANSFER '\
+BACKEND_SEP.join(
[str(self._sockno),
'REQUEST_BLOCK',
str(ct)]))
if res == '':
# complete failure, hard reset position and retry
self._count = 0
self.seek(self._pos)
continue
if int(res) == ct:
if (self._count >= 5) and (self._tsize < self._tmax):
# multiple successful transfers, bump transfer limit
self._count = 0
self._tsize += self._step
else:
if int(res) == -1:
# complete failure, hard reset position and retry
self._count = 0
self.seek(self._pos)
continue
# partial failure, reset counter and drop transfer limit
ct = int(res)
self._count = 0
self._tsize -= 2 * self._step
if self._tsize < self._step:
self._tsize = self._step
# append data and move position
buff += self.ftsock.recv(ct)
self._pos += ct
return buff
def open(self, type='r'):
"""Program.open(type='r') -> file or FileTransfer object"""
if type != 'r':
raise MythFileError(MythError.FILE_FAILED_WRITE, self.filename,
'Program () objects cannot be opened for writing.')
if self.recstatus not in (self.rsRecording, self.rsRecorded):
raise MythFileEror(MythError.FILE_FAILED_READ, '',
'Program () does not exist for access.')
if self.filename is None:
return self._openXML()
else:
return self._openProto()
def seek(self, offset, whence=0):
"""
FileTransfer.seek(offset, whence=0) -> None
Seek 'offset' number of bytes
whence == 0 - from start of file
1 - from current position
2 - from end of file
"""
if whence == 0:
if offset < 0:
offset = 0
elif offset > self._size:
offset = self._size
elif whence == 1:
if offset + self._pos < 0:
offset = -self._pos
elif offset + self._pos > self._size:
offset = self._size - self._pos
elif whence == 2:
if offset > 0:
offset = 0
elif offset < -self._size:
offset = -self._size
whence = 0
offset = self._size + offset
res = self.backendCommand('QUERY_FILETRANSFER '\
+BACKEND_SEP.join(
[str(self._sockno),'SEEK',
str(offset),
str(whence),
str(self._pos)])\
).split(BACKEND_SEP)
if res[0] == '-1':
raise MythFileError(MythError.FILE_FAILED_SEEK, \
str(self), offset, whence)
self._pos = int(res[0])
def ftopen(file, mode, forceremote=False, nooverwrite=False, db=None, \
chanid=None, starttime=None, download=False):
"""
ftopen(file, mode, forceremote=False, nooverwrite=False, db=None)
-> FileTransfer object
-> file object
Method will attempt to open file locally, falling back to remote access
over mythprotocol if necessary.
'forceremote' will force a FileTransfer object if possible.
'file' takes a standard MythURI:
myth://<group>@<host>:<port>/<path>
'mode' takes a 'r' or 'w'
'nooverwrite' will refuse to open a file writable,
if a local file is found.
"""
db = DBCache(db)
log = MythLog('Python File Transfer', db=db)
reuri = re.compile(\
'myth://((?P<group>.*)@)?(?P<host>[\[\]a-zA-Z0-9_\-\.]*)(:[0-9]*)?/(?P<file>.*)')
reip = re.compile('(?:\d{1,3}\.){3}\d{1,3}')
if mode not in ('r', 'w'):
raise TypeError("File I/O must be of type 'r' or 'w'")
if chanid and starttime:
protoopen = lambda host, file, storagegroup: \
RecordFileTransfer(host, file, storagegroup,\
mode, chanid, starttime, db)
elif download:
protoopen = lambda host, lfile, storagegroup: \
DownloadFileTransfer(host, lfile, storagegroup, \
mode, file, db)
else:
protoopen = lambda host, file, storagegroup: \
FileTransfer(host, file, storagegroup, mode, db)
# process URI (myth://<group>@<host>[:<port>]/<path/to/file>)
match = reuri.match(file)
if match is None:
raise MythError('Invalid FileTransfer input string: ' + file)
host = match.group('host')
filename = match.group('file')
sgroup = match.group('group')
if sgroup is None:
sgroup = 'Default'
# get full system name
host = host.strip('[]')
if reip.match(host) or check_ipv6(host):
host = db._gethostfromaddr(host)
# user forced to remote access
if forceremote:
if (mode == 'w') and (filename.find('/') != -1):
raise MythFileError(MythError.FILE_FAILED_WRITE, file,
'attempting remote write outside base path')
if nooverwrite and FileOps(host, db=db).fileExists(filename, sgroup):
raise MythFileError(MythError.FILE_FAILED_WRITE, file,
'refusing to overwrite existing file')
return protoopen(host, filename, sgroup)
if mode == 'w':
# check for pre-existing file
path = FileOps(host, db=db).fileExists(filename, sgroup)
sgs = list(db.getStorageGroup(groupname=sgroup))
if path is not None:
if nooverwrite:
raise MythFileError(MythError.FILE_FAILED_WRITE, file,
'refusing to overwrite existing file')
for sg in sgs:
if sg.dirname in path:
if sg.local:
return open(sg.dirname + filename, mode)
else:
return protoopen(host, filename, sgroup)
# prefer local storage for new files
for i, v in reversed(list(enumerate(sgs))):
if not v.local:
sgs.pop(i)
else:
st = os.statvfs(v.dirname)
v.free = st[0] * st[3]
if len(sgs) > 0:
# choose path with most free space
sg = sorted(sgs, key=lambda sg: sg.free, reverse=True)[0]
# create folder if it does not exist
if filename.find('/') != -1:
path = sg.dirname + filename.rsplit('/', 1)[0]
if not os.access(path, os.F_OK):
os.makedirs(path)
log(log.FILE, log.INFO, 'Opening local file (w)',
sg.dirname + filename)
return open(sg.dirname + filename, mode)
# fallback to remote write
else:
if filename.find('/') != -1:
raise MythFileError(
MythError.FILE_FAILED_WRITE, file,
'attempting remote write outside base path')
return protoopen(host, filename, sgroup)
else:
# search for file in local directories
sg = findfile(filename, sgroup, db)
if sg is not None:
# file found, open local
log(log.FILE, log.INFO, 'Opening local file (r)',
sg.dirname + filename)
return open(sg.dirname + filename, mode)
else:
# file not found, open remote
return protoopen(host, filename, sgroup)