Ejemplo n.º 1
0
    def __init__(self, backend=None, port=None, db=None):
        if backend and port:
            XMLConnection.__init__(self, backend, port)
            self.db = db
            return

        self.db = DBCache(db)
        self.log = MythLog('Python XML Connection')
        if backend is None:
            # use master backend
            backend = self.db.settings.NULL.MasterServerIP
        if re.match('(?:\d{1,3}\.){3}\d{1,3}',backend) or \
                    check_ipv6(backend):
            # process ip address
            host = self.db._gethostfromaddr(backend)
            self.host = backend
            self.port = int(self.db.settings[host].BackendStatusPort)
        else:
            # assume given a hostname
            self.host = backend
            self.port = int(self.db.settings[self.host].BackendStatusPort)
            if not self.port:
                # try a truncated hostname
                self.host = backend.split('.')[0]
                self.port = int(self.db.setting[self.host].BackendStatusPort)
                if not self.port:
                    raise MythDBError(MythError.DB_SETTING, 
                                        backend+': BackendStatusPort')
Ejemplo n.º 2
0
    def __init__(self, backend=None, port=None, db=None):
        if backend and port:
            XMLConnection.__init__(self, backend, port)
            self.db = db
            return

        self.db = DBCache(db)
        self.log = MythLog('Python XML Connection')
        if backend is None:
            # use master backend
            backend = self.db.settings.NULL.MasterServerIP
        if re.match('(?:\d{1,3}\.){3}\d{1,3}',backend) or \
                    check_ipv6(backend):
            # process ip address
            with self.db.cursor(self.log) as cursor:
                if cursor.execute("""SELECT hostname FROM settings
                                     WHERE value='BackendServerIP'
                                     AND data=%s""", backend) == 0:
                    raise MythDBError(MythError.DB_SETTING, 
                                        backend+': BackendServerIP')
                host = cursor.fetchone()[0]
            self.host = backend
            self.port = int(self.db.settings[host].BackendStatusPort)
        else:
            # assume given a hostname
            self.host = backend
            self.port = int(self.db.settings[self.host].BackendStatusPort)
            if not self.port:
                # try a truncated hostname
                self.host = backend.split('.')[0]
                self.port = int(self.db.setting[self.host].BackendStatusPort)
                if not self.port:
                    raise MythDBError(MythError.DB_SETTING, 
                                        backend+': BackendStatusPort')
Ejemplo n.º 3
0
    def __init__(self, backend=None, blockshutdown=False, events=False, db=None):
        self.db = DBCache(db)
        self.log = MythLog(self.logmodule, db=self.db)
        self.hostname = None

        self.sendcommands = True
        self.blockshutdown = blockshutdown
        self.receiveevents = events

        if backend is None:
            # no backend given, use master
            self.host = self.db.settings.NULL.MasterServerIP
            self.hostname = self.db._gethostfromaddr(self.host)

        else:
            backend = backend.strip('[]')
            query = "SELECT hostname FROM settings WHERE value=? AND data=?"
            if self._reip.match(backend):
                # given backend is IP address
                self.host = backend
                self.hostname = self.db._gethostfromaddr(
                                            backend, 'BackendServerIP')
            elif check_ipv6(backend):
                # given backend is IPv6 address
                self.host = backend
                self.hostname = self.db._gethostfromaddr(
                                            backend, 'BackendServerIP6')
            else:
                # given backend is hostname, pull address from database
                self.hostname = backend
                self.host = self.db._getpreferredaddr(backend)

        # lookup port from database
        self.port = int(self.db.settings[self.hostname].BackendServerPort)
        if not self.port:
            raise MythDBError(MythError.DB_SETTING, 'BackendServerPort',
                                            self.port)

        self._ident = '%s:%d' % (self.host, self.port)
        if self._ident in self._shared:
            # existing connection found
            self._conn = self._shared[self._ident]
            if self.sendcommands:
                if self._conn.command is None:
                    self._conn.command = self._newcmdconn()
                elif self.blockshutdown:
                    # upref block of shutdown
                    # issue command to backend if needed
                    self._conn.blockshutdown += 1
                    if self._conn.blockshutdown == 1:
                        self._conn.command.blockShutdown()
            if self.receiveevents:
                if self._conn.event is None:
                    self._conn.event = self._neweventconn()
        else:
            # no existing connection, create new
            self._conn = self._ConnHolder()

            if self.sendcommands:
                self._conn.command = self._newcmdconn()
                if self.blockshutdown:
                    self._conn.blockshutdown = 1
            if self.receiveevents:
                self._conn.event = self._neweventconn()

            self._shared[self._ident] = self._conn

        self._events = self._listhandlers()
        for func in self._events:
            self.registerevent(func)
Ejemplo n.º 4
0
def ftopen(file, mode, forceremote=False, nooverwrite=False, db=None, \
                       chanid=None, starttime=None, download=False):
    """
    ftopen(file, mode, forceremote=False, nooverwrite=False, db=None)
                                        -> FileTransfer object
                                        -> file object
    Method will attempt to open file locally, falling back to remote access
            over mythprotocol if necessary.
    'forceremote' will force a FileTransfer object if possible.
    'file' takes a standard MythURI:
                myth://<group>@<host>:<port>/<path>
    'mode' takes a 'r' or 'w'
    'nooverwrite' will refuse to open a file writable,
                if a local file is found.
    """
    db = DBCache(db)
    log = MythLog('Python File Transfer', db=db)
    reuri = re.compile(\
        'myth://((?P<group>.*)@)?(?P<host>[\[\]a-zA-Z0-9_\-\.]*)(:[0-9]*)?/(?P<file>.*)')
    reip = re.compile('(?:\d{1,3}\.){3}\d{1,3}')

    if mode not in ('r','w'):
        raise TypeError("File I/O must be of type 'r' or 'w'")

    if chanid and starttime:
        protoopen = lambda host, file, storagegroup: \
                      RecordFileTransfer(host, file, storagegroup,\
                                         mode, chanid, starttime, db)
    elif download:
        protoopen = lambda host, lfile, storagegroup: \
                      DownloadFileTransfer(host, lfile, storagegroup, \
                                           mode, file, db)
    else:
        protoopen = lambda host, file, storagegroup: \
                      FileTransfer(host, file, storagegroup, mode, db)

    # process URI (myth://<group>@<host>[:<port>]/<path/to/file>)
    match = reuri.match(file)
    if match is None:
        raise MythError('Invalid FileTransfer input string: '+file)
    host = match.group('host')
    filename = match.group('file')
    sgroup = match.group('group')
    if sgroup is None:
        sgroup = 'Default'

    # get full system name
    host = host.strip('[]')
    if reip.match(host) or check_ipv6(host):
        host = db._gethostfromaddr(host)

    # user forced to remote access
    if forceremote:
        if (mode == 'w') and (filename.find('/') != -1):
            raise MythFileError(MythError.FILE_FAILED_WRITE, file,
                                'attempting remote write outside base path')
        if nooverwrite and FileOps(host, db=db).fileExists(filename, sgroup):
            raise MythFileError(MythError.FILE_FAILED_WRITE, file,
                                'refusing to overwrite existing file')
        return protoopen(host, filename, sgroup)

    if mode == 'w':
        # check for pre-existing file
        path = FileOps(host, db=db).fileExists(filename, sgroup)
        sgs = list(db.getStorageGroup(groupname=sgroup))
        if path is not None:
            if nooverwrite:
                raise MythFileError(MythError.FILE_FAILED_WRITE, file,
                                'refusing to overwrite existing file')
            for sg in sgs:
                if sg.dirname in path:
                    if sg.local:
                        return open(sg.dirname+filename, mode)
                    else:
                        return protoopen(host, filename, sgroup)

        # prefer local storage for new files
        for i,v in reversed(list(enumerate(sgs))):
            if not v.local:
                sgs.pop(i)
            else:
                st = os.statvfs(v.dirname)
                v.free = st[0]*st[3]
        if len(sgs) > 0:
            # choose path with most free space
            sg = sorted(sgs, key=lambda sg: sg.free, reverse=True)[0]
            # create folder if it does not exist
            if filename.find('/') != -1:
                path = sg.dirname+filename.rsplit('/',1)[0]
                if not os.access(path, os.F_OK):
                    os.makedirs(path)
            log(log.FILE, log.INFO, 'Opening local file (w)',
                sg.dirname+filename)
            return open(sg.dirname+filename, mode)

        # fallback to remote write
        else:
            if filename.find('/') != -1:
                raise MythFileError(MythError.FILE_FAILED_WRITE, file,
                                'attempting remote write outside base path')
            return protoopen(host, filename, sgroup)
    else:
        # search for file in local directories
        sg = findfile(filename, sgroup, db)
        if sg is not None:
            # file found, open local
            log(log.FILE, log.INFO, 'Opening local file (r)',
                sg.dirname+filename)
            return open(sg.dirname+filename, mode)
        else:
        # file not found, open remote
            return protoopen(host, filename, sgroup)
Ejemplo n.º 5
0
    def __init__(self, backend=None, blockshutdown=False, events=False, db=None):
        self.db = DBCache(db)
        self.log = MythLog(self.logmodule, db=self.db)
        self.hostname = None

        self.sendcommands = True
        self.blockshutdown = blockshutdown
        self.receiveevents = events

        if backend is None:
            # no backend given, use master
            self.host = self.db.settings.NULL.MasterServerIP

        else:
            backend = backend.strip("[]")
            if self._reip.match(backend):
                # given backend is IP address
                self.host = backend
            elif check_ipv6(backend):
                # given backend is IPv6 address
                self.host = backend
            else:
                # given backend is hostname, pull address from database
                self.hostname = backend
                self.host = self.db.settings[backend].BackendServerIP
                if not self.host:
                    raise MythDBError(MythError.DB_SETTING, "BackendServerIP", backend)

        if self.hostname is None:
            # reverse lookup hostname from address
            with self.db.cursor(self.log) as cursor:
                if (
                    cursor.execute(
                        """SELECT hostname FROM settings
                                     WHERE value='BackendServerIP'
                                     AND data=?""",
                        [self.host],
                    )
                    == 0
                ):
                    # no match found
                    raise MythDBError(MythError.DB_SETTING, "BackendServerIP", self.host)
                self.hostname = cursor.fetchone()[0]

        # lookup port from database
        self.port = int(self.db.settings[self.hostname].BackendServerPort)
        if not self.port:
            raise MythDBError(MythError.DB_SETTING, "BackendServerPort", self.port)

        self._ident = "%s:%d" % (self.host, self.port)
        if self._ident in self._shared:
            # existing connection found
            self._conn = self._shared[self._ident]
            if self.sendcommands:
                if self._conn.command is None:
                    self._conn.command = self._newcmdconn()
                elif self.blockshutdown:
                    # upref block of shutdown
                    # issue command to backend if needed
                    self._conn.blockshutdown += 1
                    if self._conn.blockshutdown == 1:
                        self._conn.command.blockShutdown()
            if self.receiveevents:
                if self._conn.event is None:
                    self._conn.event = self._neweventconn()
        else:
            # no existing connection, create new
            self._conn = self._ConnHolder()

            if self.sendcommands:
                self._conn.command = self._newcmdconn()
                if self.blockshutdown:
                    self._conn.blockshutdown = 1
            if self.receiveevents:
                self._conn.event = self._neweventconn()

            self._shared[self._ident] = self._conn

        self._events = self._listhandlers()
        for func in self._events:
            self.registerevent(func)
Ejemplo n.º 6
0
def ftopen(file, mode, forceremote=False, nooverwrite=False, db=None, chanid=None, starttime=None, download=False):
    """
    ftopen(file, mode, forceremote=False, nooverwrite=False, db=None)
                                        -> FileTransfer object
                                        -> file object
    Method will attempt to open file locally, falling back to remote access
            over mythprotocol if necessary.
    'forceremote' will force a FileTransfer object if possible.
    'file' takes a standard MythURI:
                myth://<group>@<host>:<port>/<path>
    'mode' takes a 'r' or 'w'
    'nooverwrite' will refuse to open a file writable,
                if a local file is found.
    """
    db = DBCache(db)
    log = MythLog("Python File Transfer", db=db)
    reuri = re.compile("myth://((?P<group>.*)@)?(?P<host>[\[\]a-zA-Z0-9_\-\.]*)(:[0-9]*)?/(?P<file>.*)")
    reip = re.compile("(?:\d{1,3}\.){3}\d{1,3}")

    if mode not in ("r", "w"):
        raise TypeError("File I/O must be of type 'r' or 'w'")

    if chanid and starttime:
        protoopen = lambda host, file, storagegroup: RecordFileTransfer(
            host, file, storagegroup, mode, chanid, starttime, db
        )
    elif download:
        protoopen = lambda host, lfile, storagegroup: DownloadFileTransfer(host, lfile, storagegroup, mode, file, db)
    else:
        protoopen = lambda host, file, storagegroup: FileTransfer(host, file, storagegroup, mode, db)

    # process URI (myth://<group>@<host>[:<port>]/<path/to/file>)
    match = reuri.match(file)
    if match is None:
        raise MythError("Invalid FileTransfer input string: " + file)
    host = match.group("host")
    filename = match.group("file")
    sgroup = match.group("group")
    if sgroup is None:
        sgroup = "Default"

    # get full system name
    host = host.strip("[]")
    if reip.match(host) or check_ipv6(host):
        with db.cursor(log) as cursor:
            if (
                cursor.execute(
                    """SELECT hostname FROM settings
                                 WHERE value='BackendServerIP'
                                 AND data=%s""",
                    host,
                )
                == 0
            ):
                raise MythDBError(MythError.DB_SETTING, "BackendServerIP", backend)
            host = cursor.fetchone()[0]

    # user forced to remote access
    if forceremote:
        if (mode == "w") and (filename.find("/") != -1):
            raise MythFileError(MythError.FILE_FAILED_WRITE, file, "attempting remote write outside base path")
        if nooverwrite and FileOps(host, db=db).fileExists(filename, sgroup):
            raise MythFileError(MythError.FILE_FAILED_WRITE, file, "refusing to overwrite existing file")
        return protoopen(host, filename, sgroup)

    if mode == "w":
        # check for pre-existing file
        path = FileOps(host, db=db).fileExists(filename, sgroup)
        sgs = list(db.getStorageGroup(groupname=sgroup))
        if path is not None:
            if nooverwrite:
                raise MythFileError(MythError.FILE_FAILED_WRITE, file, "refusing to overwrite existing file")
            for sg in sgs:
                if sg.dirname in path:
                    if sg.local:
                        return open(sg.dirname + filename, mode)
                    else:
                        return protoopen(host, filename, sgroup)

        # prefer local storage for new files
        for i, v in reversed(list(enumerate(sgs))):
            if not v.local:
                sgs.pop(i)
            else:
                st = os.statvfs(v.dirname)
                v.free = st[0] * st[3]
        if len(sgs) > 0:
            # choose path with most free space
            sg = sorted(sgs, key=lambda sg: sg.free, reverse=True)[0]
            # create folder if it does not exist
            if filename.find("/") != -1:
                path = sg.dirname + filename.rsplit("/", 1)[0]
                if not os.access(path, os.F_OK):
                    os.makedirs(path)
            log(log.FILE, "Opening local file (w)", sg.dirname + filename)
            return open(sg.dirname + filename, mode)

        # fallback to remote write
        else:
            if filename.find("/") != -1:
                raise MythFileError(MythError.FILE_FAILED_WRITE, file, "attempting remote write outside base path")
            return protoopen(host, filename, sgroup)
    else:
        # search for file in local directories
        sg = findfile(filename, sgroup, db)
        if sg is not None:
            # file found, open local
            log(log.FILE, "Opening local file (r)", sg.dirname + filename)
            return open(sg.dirname + filename, mode)
        else:
            # file not found, open remote
            return protoopen(host, filename, sgroup)