def __init__(self, backend=None, port=None, db=None): if backend and port: XMLConnection.__init__(self, backend, port) self.db = db return self.db = DBCache(db) self.log = MythLog('Python XML Connection') if backend is None: # use master backend backend = self.db.settings.NULL.MasterServerIP if re.match('(?:\d{1,3}\.){3}\d{1,3}',backend) or \ check_ipv6(backend): # process ip address host = self.db._gethostfromaddr(backend) self.host = backend self.port = int(self.db.settings[host].BackendStatusPort) else: # assume given a hostname self.host = backend self.port = int(self.db.settings[self.host].BackendStatusPort) if not self.port: # try a truncated hostname self.host = backend.split('.')[0] self.port = int(self.db.setting[self.host].BackendStatusPort) if not self.port: raise MythDBError(MythError.DB_SETTING, backend+': BackendStatusPort')
def __init__(self, backend=None, port=None, db=None): if backend and port: XMLConnection.__init__(self, backend, port) self.db = db return self.db = DBCache(db) self.log = MythLog('Python XML Connection') if backend is None: # use master backend backend = self.db.settings.NULL.MasterServerIP if re.match('(?:\d{1,3}\.){3}\d{1,3}',backend) or \ check_ipv6(backend): # process ip address with self.db.cursor(self.log) as cursor: if cursor.execute("""SELECT hostname FROM settings WHERE value='BackendServerIP' AND data=%s""", backend) == 0: raise MythDBError(MythError.DB_SETTING, backend+': BackendServerIP') host = cursor.fetchone()[0] self.host = backend self.port = int(self.db.settings[host].BackendStatusPort) else: # assume given a hostname self.host = backend self.port = int(self.db.settings[self.host].BackendStatusPort) if not self.port: # try a truncated hostname self.host = backend.split('.')[0] self.port = int(self.db.setting[self.host].BackendStatusPort) if not self.port: raise MythDBError(MythError.DB_SETTING, backend+': BackendStatusPort')
def __init__(self, backend=None, blockshutdown=False, events=False, db=None): self.db = DBCache(db) self.log = MythLog(self.logmodule, db=self.db) self.hostname = None self.sendcommands = True self.blockshutdown = blockshutdown self.receiveevents = events if backend is None: # no backend given, use master self.host = self.db.settings.NULL.MasterServerIP self.hostname = self.db._gethostfromaddr(self.host) else: backend = backend.strip('[]') query = "SELECT hostname FROM settings WHERE value=? AND data=?" if self._reip.match(backend): # given backend is IP address self.host = backend self.hostname = self.db._gethostfromaddr( backend, 'BackendServerIP') elif check_ipv6(backend): # given backend is IPv6 address self.host = backend self.hostname = self.db._gethostfromaddr( backend, 'BackendServerIP6') else: # given backend is hostname, pull address from database self.hostname = backend self.host = self.db._getpreferredaddr(backend) # lookup port from database self.port = int(self.db.settings[self.hostname].BackendServerPort) if not self.port: raise MythDBError(MythError.DB_SETTING, 'BackendServerPort', self.port) self._ident = '%s:%d' % (self.host, self.port) if self._ident in self._shared: # existing connection found self._conn = self._shared[self._ident] if self.sendcommands: if self._conn.command is None: self._conn.command = self._newcmdconn() elif self.blockshutdown: # upref block of shutdown # issue command to backend if needed self._conn.blockshutdown += 1 if self._conn.blockshutdown == 1: self._conn.command.blockShutdown() if self.receiveevents: if self._conn.event is None: self._conn.event = self._neweventconn() else: # no existing connection, create new self._conn = self._ConnHolder() if self.sendcommands: self._conn.command = self._newcmdconn() if self.blockshutdown: self._conn.blockshutdown = 1 if self.receiveevents: self._conn.event = self._neweventconn() self._shared[self._ident] = self._conn self._events = self._listhandlers() for func in self._events: self.registerevent(func)
def ftopen(file, mode, forceremote=False, nooverwrite=False, db=None, \ chanid=None, starttime=None, download=False): """ ftopen(file, mode, forceremote=False, nooverwrite=False, db=None) -> FileTransfer object -> file object Method will attempt to open file locally, falling back to remote access over mythprotocol if necessary. 'forceremote' will force a FileTransfer object if possible. 'file' takes a standard MythURI: myth://<group>@<host>:<port>/<path> 'mode' takes a 'r' or 'w' 'nooverwrite' will refuse to open a file writable, if a local file is found. """ db = DBCache(db) log = MythLog('Python File Transfer', db=db) reuri = re.compile(\ 'myth://((?P<group>.*)@)?(?P<host>[\[\]a-zA-Z0-9_\-\.]*)(:[0-9]*)?/(?P<file>.*)') reip = re.compile('(?:\d{1,3}\.){3}\d{1,3}') if mode not in ('r','w'): raise TypeError("File I/O must be of type 'r' or 'w'") if chanid and starttime: protoopen = lambda host, file, storagegroup: \ RecordFileTransfer(host, file, storagegroup,\ mode, chanid, starttime, db) elif download: protoopen = lambda host, lfile, storagegroup: \ DownloadFileTransfer(host, lfile, storagegroup, \ mode, file, db) else: protoopen = lambda host, file, storagegroup: \ FileTransfer(host, file, storagegroup, mode, db) # process URI (myth://<group>@<host>[:<port>]/<path/to/file>) match = reuri.match(file) if match is None: raise MythError('Invalid FileTransfer input string: '+file) host = match.group('host') filename = match.group('file') sgroup = match.group('group') if sgroup is None: sgroup = 'Default' # get full system name host = host.strip('[]') if reip.match(host) or check_ipv6(host): host = db._gethostfromaddr(host) # user forced to remote access if forceremote: if (mode == 'w') and (filename.find('/') != -1): raise MythFileError(MythError.FILE_FAILED_WRITE, file, 'attempting remote write outside base path') if nooverwrite and FileOps(host, db=db).fileExists(filename, sgroup): raise MythFileError(MythError.FILE_FAILED_WRITE, file, 'refusing to overwrite existing file') return protoopen(host, filename, sgroup) if mode == 'w': # check for pre-existing file path = FileOps(host, db=db).fileExists(filename, sgroup) sgs = list(db.getStorageGroup(groupname=sgroup)) if path is not None: if nooverwrite: raise MythFileError(MythError.FILE_FAILED_WRITE, file, 'refusing to overwrite existing file') for sg in sgs: if sg.dirname in path: if sg.local: return open(sg.dirname+filename, mode) else: return protoopen(host, filename, sgroup) # prefer local storage for new files for i,v in reversed(list(enumerate(sgs))): if not v.local: sgs.pop(i) else: st = os.statvfs(v.dirname) v.free = st[0]*st[3] if len(sgs) > 0: # choose path with most free space sg = sorted(sgs, key=lambda sg: sg.free, reverse=True)[0] # create folder if it does not exist if filename.find('/') != -1: path = sg.dirname+filename.rsplit('/',1)[0] if not os.access(path, os.F_OK): os.makedirs(path) log(log.FILE, log.INFO, 'Opening local file (w)', sg.dirname+filename) return open(sg.dirname+filename, mode) # fallback to remote write else: if filename.find('/') != -1: raise MythFileError(MythError.FILE_FAILED_WRITE, file, 'attempting remote write outside base path') return protoopen(host, filename, sgroup) else: # search for file in local directories sg = findfile(filename, sgroup, db) if sg is not None: # file found, open local log(log.FILE, log.INFO, 'Opening local file (r)', sg.dirname+filename) return open(sg.dirname+filename, mode) else: # file not found, open remote return protoopen(host, filename, sgroup)
def __init__(self, backend=None, blockshutdown=False, events=False, db=None): self.db = DBCache(db) self.log = MythLog(self.logmodule, db=self.db) self.hostname = None self.sendcommands = True self.blockshutdown = blockshutdown self.receiveevents = events if backend is None: # no backend given, use master self.host = self.db.settings.NULL.MasterServerIP else: backend = backend.strip("[]") if self._reip.match(backend): # given backend is IP address self.host = backend elif check_ipv6(backend): # given backend is IPv6 address self.host = backend else: # given backend is hostname, pull address from database self.hostname = backend self.host = self.db.settings[backend].BackendServerIP if not self.host: raise MythDBError(MythError.DB_SETTING, "BackendServerIP", backend) if self.hostname is None: # reverse lookup hostname from address with self.db.cursor(self.log) as cursor: if ( cursor.execute( """SELECT hostname FROM settings WHERE value='BackendServerIP' AND data=?""", [self.host], ) == 0 ): # no match found raise MythDBError(MythError.DB_SETTING, "BackendServerIP", self.host) self.hostname = cursor.fetchone()[0] # lookup port from database self.port = int(self.db.settings[self.hostname].BackendServerPort) if not self.port: raise MythDBError(MythError.DB_SETTING, "BackendServerPort", self.port) self._ident = "%s:%d" % (self.host, self.port) if self._ident in self._shared: # existing connection found self._conn = self._shared[self._ident] if self.sendcommands: if self._conn.command is None: self._conn.command = self._newcmdconn() elif self.blockshutdown: # upref block of shutdown # issue command to backend if needed self._conn.blockshutdown += 1 if self._conn.blockshutdown == 1: self._conn.command.blockShutdown() if self.receiveevents: if self._conn.event is None: self._conn.event = self._neweventconn() else: # no existing connection, create new self._conn = self._ConnHolder() if self.sendcommands: self._conn.command = self._newcmdconn() if self.blockshutdown: self._conn.blockshutdown = 1 if self.receiveevents: self._conn.event = self._neweventconn() self._shared[self._ident] = self._conn self._events = self._listhandlers() for func in self._events: self.registerevent(func)
def ftopen(file, mode, forceremote=False, nooverwrite=False, db=None, chanid=None, starttime=None, download=False): """ ftopen(file, mode, forceremote=False, nooverwrite=False, db=None) -> FileTransfer object -> file object Method will attempt to open file locally, falling back to remote access over mythprotocol if necessary. 'forceremote' will force a FileTransfer object if possible. 'file' takes a standard MythURI: myth://<group>@<host>:<port>/<path> 'mode' takes a 'r' or 'w' 'nooverwrite' will refuse to open a file writable, if a local file is found. """ db = DBCache(db) log = MythLog("Python File Transfer", db=db) reuri = re.compile("myth://((?P<group>.*)@)?(?P<host>[\[\]a-zA-Z0-9_\-\.]*)(:[0-9]*)?/(?P<file>.*)") reip = re.compile("(?:\d{1,3}\.){3}\d{1,3}") if mode not in ("r", "w"): raise TypeError("File I/O must be of type 'r' or 'w'") if chanid and starttime: protoopen = lambda host, file, storagegroup: RecordFileTransfer( host, file, storagegroup, mode, chanid, starttime, db ) elif download: protoopen = lambda host, lfile, storagegroup: DownloadFileTransfer(host, lfile, storagegroup, mode, file, db) else: protoopen = lambda host, file, storagegroup: FileTransfer(host, file, storagegroup, mode, db) # process URI (myth://<group>@<host>[:<port>]/<path/to/file>) match = reuri.match(file) if match is None: raise MythError("Invalid FileTransfer input string: " + file) host = match.group("host") filename = match.group("file") sgroup = match.group("group") if sgroup is None: sgroup = "Default" # get full system name host = host.strip("[]") if reip.match(host) or check_ipv6(host): with db.cursor(log) as cursor: if ( cursor.execute( """SELECT hostname FROM settings WHERE value='BackendServerIP' AND data=%s""", host, ) == 0 ): raise MythDBError(MythError.DB_SETTING, "BackendServerIP", backend) host = cursor.fetchone()[0] # user forced to remote access if forceremote: if (mode == "w") and (filename.find("/") != -1): raise MythFileError(MythError.FILE_FAILED_WRITE, file, "attempting remote write outside base path") if nooverwrite and FileOps(host, db=db).fileExists(filename, sgroup): raise MythFileError(MythError.FILE_FAILED_WRITE, file, "refusing to overwrite existing file") return protoopen(host, filename, sgroup) if mode == "w": # check for pre-existing file path = FileOps(host, db=db).fileExists(filename, sgroup) sgs = list(db.getStorageGroup(groupname=sgroup)) if path is not None: if nooverwrite: raise MythFileError(MythError.FILE_FAILED_WRITE, file, "refusing to overwrite existing file") for sg in sgs: if sg.dirname in path: if sg.local: return open(sg.dirname + filename, mode) else: return protoopen(host, filename, sgroup) # prefer local storage for new files for i, v in reversed(list(enumerate(sgs))): if not v.local: sgs.pop(i) else: st = os.statvfs(v.dirname) v.free = st[0] * st[3] if len(sgs) > 0: # choose path with most free space sg = sorted(sgs, key=lambda sg: sg.free, reverse=True)[0] # create folder if it does not exist if filename.find("/") != -1: path = sg.dirname + filename.rsplit("/", 1)[0] if not os.access(path, os.F_OK): os.makedirs(path) log(log.FILE, "Opening local file (w)", sg.dirname + filename) return open(sg.dirname + filename, mode) # fallback to remote write else: if filename.find("/") != -1: raise MythFileError(MythError.FILE_FAILED_WRITE, file, "attempting remote write outside base path") return protoopen(host, filename, sgroup) else: # search for file in local directories sg = findfile(filename, sgroup, db) if sg is not None: # file found, open local log(log.FILE, "Opening local file (r)", sg.dirname + filename) return open(sg.dirname + filename, mode) else: # file not found, open remote return protoopen(host, filename, sgroup)
def __init__(self, backend=None, blockshutdown=False, events=False, db=None): self.db = DBCache(db) self.log = MythLog(self.logmodule, db=self.db) self.hostname = None self.sendcommands = True self.blockshutdown = blockshutdown self.receiveevents = events if backend is None: # no backend given, use master self.host = self.db.settings.NULL.MasterServerIP else: backend = backend.strip('[]') if self._reip.match(backend): # given backend is IP address self.host = backend elif check_ipv6(backend): # given backend is IPv6 address self.host = backend else: # given backend is hostname, pull address from database self.hostname = backend self.host = self.db.settings[backend].BackendServerIP if not self.host: raise MythDBError(MythError.DB_SETTING, 'BackendServerIP', backend) if self.hostname is None: # reverse lookup hostname from address with self.db.cursor(self.log) as cursor: if cursor.execute( """SELECT hostname FROM settings WHERE value='BackendServerIP' AND data=?""", [self.host]) == 0: # no match found raise MythDBError(MythError.DB_SETTING, 'BackendServerIP', self.host) self.hostname = cursor.fetchone()[0] # lookup port from database self.port = int(self.db.settings[self.hostname].BackendServerPort) if not self.port: raise MythDBError(MythError.DB_SETTING, 'BackendServerPort', self.port) self._ident = '%s:%d' % (self.host, self.port) if self._ident in self._shared: # existing connection found self._conn = self._shared[self._ident] if self.sendcommands: if self._conn.command is None: self._conn.command = self._newcmdconn() elif self.blockshutdown: # upref block of shutdown # issue command to backend if needed self._conn.blockshutdown += 1 if self._conn.blockshutdown == 1: self._conn.command.blockShutdown() if self.receiveevents: if self._conn.event is None: self._conn.event = self._neweventconn() else: # no existing connection, create new self._conn = self._ConnHolder() if self.sendcommands: self._conn.command = self._newcmdconn() if self.blockshutdown: self._conn.blockshutdown = 1 if self.receiveevents: self._conn.event = self._neweventconn() self._shared[self._ident] = self._conn self._events = self._listhandlers() for func in self._events: self.registerevent(func)
def ftopen(file, mode, forceremote=False, nooverwrite=False, db=None, \ chanid=None, starttime=None, download=False): """ ftopen(file, mode, forceremote=False, nooverwrite=False, db=None) -> FileTransfer object -> file object Method will attempt to open file locally, falling back to remote access over mythprotocol if necessary. 'forceremote' will force a FileTransfer object if possible. 'file' takes a standard MythURI: myth://<group>@<host>:<port>/<path> 'mode' takes a 'r' or 'w' 'nooverwrite' will refuse to open a file writable, if a local file is found. """ db = DBCache(db) log = MythLog('Python File Transfer', db=db) reuri = re.compile(\ 'myth://((?P<group>.*)@)?(?P<host>[\[\]a-zA-Z0-9_\-\.]*)(:[0-9]*)?/(?P<file>.*)') reip = re.compile('(?:\d{1,3}\.){3}\d{1,3}') if mode not in ('r', 'w'): raise TypeError("File I/O must be of type 'r' or 'w'") if chanid and starttime: protoopen = lambda host, file, storagegroup: \ RecordFileTransfer(host, file, storagegroup,\ mode, chanid, starttime, db) elif download: protoopen = lambda host, lfile, storagegroup: \ DownloadFileTransfer(host, lfile, storagegroup, \ mode, file, db) else: protoopen = lambda host, file, storagegroup: \ FileTransfer(host, file, storagegroup, mode, db) # process URI (myth://<group>@<host>[:<port>]/<path/to/file>) match = reuri.match(file) if match is None: raise MythError('Invalid FileTransfer input string: ' + file) host = match.group('host') filename = match.group('file') sgroup = match.group('group') if sgroup is None: sgroup = 'Default' # get full system name host = host.strip('[]') if reip.match(host) or check_ipv6(host): with db.cursor(log) as cursor: if cursor.execute( """SELECT hostname FROM settings WHERE value='BackendServerIP' AND data=%s""", host) == 0: raise MythDBError(MythError.DB_SETTING, \ 'BackendServerIP', backend) host = cursor.fetchone()[0] # user forced to remote access if forceremote: if (mode == 'w') and (filename.find('/') != -1): raise MythFileError(MythError.FILE_FAILED_WRITE, file, 'attempting remote write outside base path') if nooverwrite and FileOps(host, db=db).fileExists(filename, sgroup): raise MythFileError(MythError.FILE_FAILED_WRITE, file, 'refusing to overwrite existing file') return protoopen(host, filename, sgroup) if mode == 'w': # check for pre-existing file path = FileOps(host, db=db).fileExists(filename, sgroup) sgs = list(db.getStorageGroup(groupname=sgroup)) if path is not None: if nooverwrite: raise MythFileError(MythError.FILE_FAILED_WRITE, file, 'refusing to overwrite existing file') for sg in sgs: if sg.dirname in path: if sg.local: return open(sg.dirname + filename, mode) else: return protoopen(host, filename, sgroup) # prefer local storage for new files for i, v in reversed(list(enumerate(sgs))): if not v.local: sgs.pop(i) else: st = os.statvfs(v.dirname) v.free = st[0] * st[3] if len(sgs) > 0: # choose path with most free space sg = sorted(sgs, key=lambda sg: sg.free, reverse=True)[0] # create folder if it does not exist if filename.find('/') != -1: path = sg.dirname + filename.rsplit('/', 1)[0] if not os.access(path, os.F_OK): os.makedirs(path) log(log.FILE, log.INFO, 'Opening local file (w)', sg.dirname + filename) return open(sg.dirname + filename, mode) # fallback to remote write else: if filename.find('/') != -1: raise MythFileError( MythError.FILE_FAILED_WRITE, file, 'attempting remote write outside base path') return protoopen(host, filename, sgroup) else: # search for file in local directories sg = findfile(filename, sgroup, db) if sg is not None: # file found, open local log(log.FILE, log.INFO, 'Opening local file (r)', sg.dirname + filename) return open(sg.dirname + filename, mode) else: # file not found, open remote return protoopen(host, filename, sgroup)