Beispiel #1
0
def _finish_convergence_service(k8s_client, options, subscription_client):
    k8s = KubeClient(k8s=k8s_client)

    access_key_id = FilePath(
        options["aws-access-key-id-path"]).getContent().strip()
    secret_access_key = FilePath(
        options["aws-secret-access-key-path"]).getContent().strip()

    aws = AWSServiceRegion(creds=AWSCredentials(
        access_key=access_key_id,
        secret_key=secret_access_key,
    ))

    Message.log(
        event=u"convergence-service:key-notification",
        key_id=access_key_id.decode("ascii"),
        secret_key_hash=sha256(secret_access_key).hexdigest().decode("ascii"),
    )

    # XXX I get to leave a ton of fields empty because I happen to know
    # they're not used in this codepath. :/ Maybe this suggests something has
    # gone wrong ...
    config = DeploymentConfiguration(
        domain=options["domain"].decode("ascii"),
        kubernetes_namespace=options["kubernetes-namespace"].decode("ascii"),
        subscription_manager_endpoint=URL.fromText(
            options["endpoint"].decode("ascii")),
        s3_access_key_id=access_key_id.decode("ascii"),
        s3_secret_key=secret_access_key.decode("ascii"),
        introducer_image=options["introducer-image"].decode("ascii"),
        storageserver_image=options["storageserver-image"].decode("ascii"),
        log_gatherer_furl=None,
        stats_gatherer_furl=None,
    )

    return TimerService(
        options["interval"],
        divert_errors_to_log(converge, u"subscription_converger"),
        config,
        subscription_client,
        k8s,
        aws,
    )
def _finish_convergence_service(
    k8s_client, options, subscription_client, reactor,
):
    k8s = KubeClient(k8s=k8s_client)

    access_key_id = FilePath(options["aws-access-key-id-path"]).getContent().strip()
    secret_access_key = FilePath(options["aws-secret-access-key-path"]).getContent().strip()

    aws = AWSServiceRegion(creds=AWSCredentials(
        access_key=access_key_id,
        secret_key=secret_access_key,
    ))

    Message.log(
        event=u"convergence-service:key-notification",
        key_id=access_key_id.decode("ascii"),
        secret_key_hash=sha256(secret_access_key).hexdigest().decode("ascii"),
    )

    config = DeploymentConfiguration(
        domain=options["domain"].decode("ascii"),
        kubernetes_namespace=options["kubernetes-namespace"].decode("ascii"),
        subscription_manager_endpoint=URL.fromText(options["endpoint"].decode("ascii")),

        s3_access_key_id=access_key_id.decode("ascii"),
        s3_secret_key=secret_access_key.decode("ascii"),

        introducer_image=options["introducer-image"].decode("ascii"),
        storageserver_image=options["storageserver-image"].decode("ascii"),

        log_gatherer_furl=options["log-gatherer-furl"],
        stats_gatherer_furl=options["stats-gatherer-furl"],
    )

    return _convergence_service(
        reactor,
        options["interval"],
        config,
        subscription_client,
        k8s,
        aws,
    )
def _finish_convergence_service(
    k8s_client, options, subscription_client, reactor,
):
    k8s = KubeClient(k8s=k8s_client)

    access_key_id = FilePath(options["aws-access-key-id-path"]).getContent().strip()
    secret_access_key = FilePath(options["aws-secret-access-key-path"]).getContent().strip()

    aws = AWSServiceRegion(creds=AWSCredentials(
        access_key=access_key_id,
        secret_key=secret_access_key,
    ))

    Message.log(
        event=u"convergence-service:key-notification",
        key_id=access_key_id.decode("ascii"),
        secret_key_hash=sha256(secret_access_key).hexdigest().decode("ascii"),
    )

    config = DeploymentConfiguration(
        domain=options["domain"].decode("ascii"),
        kubernetes_namespace=options["kubernetes-namespace"].decode("ascii"),
        subscription_manager_endpoint=URL.fromText(options["endpoint"].decode("ascii")),

        s3_access_key_id=access_key_id.decode("ascii"),
        s3_secret_key=secret_access_key.decode("ascii"),

        introducer_image=options["introducer-image"].decode("ascii"),
        storageserver_image=options["storageserver-image"].decode("ascii"),

        log_gatherer_furl=options["log-gatherer-furl"],
        stats_gatherer_furl=options["stats-gatherer-furl"],
    )

    return _convergence_service(
        reactor,
        options["interval"],
        config,
        subscription_client,
        k8s,
        aws,
    )
Beispiel #4
0
class FSItem(BackendItem):
    logCategory = 'fs_item'

    def __init__(self, object_id, parent, path, mimetype, urlbase, UPnPClass,update=False):
        self.id = object_id
        self.parent = parent
        if parent:
            parent.add_child(self,update=update)
        if mimetype == 'root':
            self.location = unicode(path)
        else:
            if mimetype == 'item' and path is None:
                path = os.path.join(parent.get_path(),unicode(self.id))
            #self.location = FilePath(unicode(path))
            self.location = FilePath(path)
        self.mimetype = mimetype
        if urlbase[-1] != '/':
            urlbase += '/'
        self.url = urlbase + str(self.id)


        if parent == None:
            parent_id = -1
        else:
            parent_id = parent.get_id()

        self.item = UPnPClass(object_id, parent_id, self.get_name())
        if isinstance(self.item, Container):
            self.item.childCount = 0
        self.child_count = 0
        self.children = []


        if mimetype in ['directory','root']:
            self.update_id = 0
            self.get_url = lambda : self.url
            self.get_path = lambda : None
            #self.item.searchable = True
            #self.item.searchClass = 'object'
            if(isinstance(self.location,FilePath) and
               self.location.isdir() == True):
                self.check_for_cover_art()
                if hasattr(self, 'cover'):
                    _,ext =  os.path.splitext(self.cover)
                    """ add the cover image extension to help clients not reacting on
                        the mimetype """
                    self.item.albumArtURI = ''.join((urlbase,str(self.id),'?cover',ext))
        else:
            self.get_url = lambda : self.url

            if self.mimetype.startswith('audio/'):
                if hasattr(parent, 'cover'):
                    _,ext =  os.path.splitext(parent.cover)
                    """ add the cover image extension to help clients not reacting on
                        the mimetype """
                    self.item.albumArtURI = ''.join((urlbase,str(self.id),'?cover',ext))

            _,host_port,_,_,_ = urlsplit(urlbase)
            if host_port.find(':') != -1:
                host,port = tuple(host_port.split(':'))
            else:
                host = host_port

            try:
                size = self.location.getsize()
            except:
                size = 0

            if mimetype != 'item':
                res = Resource('file://'+ urllib.quote(self.get_path()), 'internal:%s:%s:*' % (host,self.mimetype))
                res.size = size
                self.item.res.append(res)

            if mimetype != 'item':
                res = Resource(self.url, 'http-get:*:%s:*' % self.mimetype)
            else:
                res = Resource(self.url, 'http-get:*:*:*')

            res.size = size
            self.item.res.append(res)


            """ if this item is an image and we want to add a thumbnail for it
                we have to follow these rules:

                create a new Resource object, at least a 'http-get'
                and maybe an 'internal' one too

                for an JPG this looks like that

                res = Resource(url_for_thumbnail,
                        'http-get:*:image/jpg:%s'% ';'.join(simple_dlna_tags+('DLNA.ORG_PN=JPEG_TN',)))
                res.size = size_of_thumbnail
                self.item.res.append(res)

                and for a PNG the Resource creation is like that

                res = Resource(url_for_thumbnail,
                        'http-get:*:image/png:%s'% ';'.join(simple_dlna_tags+('DLNA.ORG_PN=PNG_TN',)))

                if not hasattr(self.item, 'attachments'):
                    self.item.attachments = {}
                self.item.attachments[key] = utils.StaticFile(filename_of_thumbnail)
            """

            if self.mimetype in ('image/jpeg', 'image/png'):
                path = self.get_path()
                thumbnail = os.path.join(os.path.dirname(path),'.thumbs',os.path.basename(path))
                if os.path.exists(thumbnail):
                    mimetype,_ = mimetypes.guess_type(thumbnail, strict=False)
                    if mimetype in ('image/jpeg','image/png'):
                        if mimetype == 'image/jpeg':
                            dlna_pn = 'DLNA.ORG_PN=JPEG_TN'
                        else:
                            dlna_pn = 'DLNA.ORG_PN=PNG_TN'

                        hash_from_path = str(id(thumbnail))
                        new_res = Resource(self.url+'?attachment='+hash_from_path,
                            'http-get:*:%s:%s' % (mimetype, ';'.join(simple_dlna_tags+(dlna_pn,))))
                        new_res.size = os.path.getsize(thumbnail)
                        self.item.res.append(new_res)
                        if not hasattr(self.item, 'attachments'):
                            self.item.attachments = {}
                        self.item.attachments[hash_from_path] = utils.StaticFile(urllib.quote(thumbnail))


            try:
                # FIXME: getmtime is deprecated in Twisted 2.6
                self.item.date = datetime.fromtimestamp(self.location.getmtime())
            except:
                self.item.date = None

    def rebuild(self, urlbase):
        #print "rebuild", self.mimetype
        if self.mimetype != 'item':
            return
        #print "rebuild for", self.get_path()
        mimetype,_ = mimetypes.guess_type(self.get_path(),strict=False)
        if mimetype == None:
            return
        self.mimetype = mimetype
        #print "rebuild", self.mimetype
        UPnPClass = classChooser(self.mimetype)
        self.item = UPnPClass(self.id, self.parent.id, self.get_name())
        if hasattr(self.parent, 'cover'):
            _,ext =  os.path.splitext(self.parent.cover)
            """ add the cover image extension to help clients not reacting on
                the mimetype """
            self.item.albumArtURI = ''.join((urlbase,str(self.id),'?cover',ext))

        _,host_port,_,_,_ = urlsplit(urlbase)
        if host_port.find(':') != -1:
            host,port = tuple(host_port.split(':'))
        else:
            host = host_port

        res = Resource('file://'+urllib.quote(self.get_path()), 'internal:%s:%s:*' % (host,self.mimetype))
        try:
            res.size = self.location.getsize()
        except:
            res.size = 0
        self.item.res.append(res)
        res = Resource(self.url, 'http-get:*:%s:*' % self.mimetype)

        try:
            res.size = self.location.getsize()
        except:
            res.size = 0
        self.item.res.append(res)

        try:
            # FIXME: getmtime is deprecated in Twisted 2.6
            self.item.date = datetime.fromtimestamp(self.location.getmtime())
        except:
            self.item.date = None

        self.parent.update_id += 1

    def check_for_cover_art(self):
        """ let's try to find in the current directory some jpg file,
            or png if the jpg search fails, and take the first one
            that comes around
        """
        try:
            jpgs = [i.path for i in self.location.children() if i.splitext()[1] in ('.jpg', '.JPG')]
            try:
                self.cover = jpgs[0]
            except IndexError:
                pngs = [i.path for i in self.location.children() if i.splitext()[1] in ('.png', '.PNG')]
                try:
                    self.cover = pngs[0]
                except IndexError:
                    return
        except UnicodeDecodeError:
            self.warning("UnicodeDecodeError - there is something wrong with a file located in %r", self.location.path)

    def remove(self):
        #print "FSItem remove", self.id, self.get_name(), self.parent
        if self.parent:
            self.parent.remove_child(self)
        del self.item

    def add_child(self, child, update=False):
        self.children.append(child)
        self.child_count += 1
        if isinstance(self.item, Container):
            self.item.childCount += 1
        if update == True:
            self.update_id += 1

    def remove_child(self, child):
        #print "remove_from %d (%s) child %d (%s)" % (self.id, self.get_name(), child.id, child.get_name())
        if child in self.children:
            self.child_count -= 1
            if isinstance(self.item, Container):
                self.item.childCount -= 1
            self.children.remove(child)
            self.update_id += 1

    def get_children(self,start=0,request_count=0):
        if request_count == 0:
            return self.children[start:]
        else:
            return self.children[start:request_count]

    def get_child_count(self):
        return self.child_count

    def get_id(self):
        return self.id

    def get_update_id(self):
        if hasattr(self, 'update_id'):
            return self.update_id
        else:
            return None

    def get_path(self):
        if isinstance( self.location,FilePath):
            return self.location.path
        else:
            self.location

    def set_path(self,path=None,extension=None):
        if path is None:
            path = self.get_path()
        if extension is not None:
            path,old_ext = os.path.splitext(path)
            path = ''.join((path,extension))
        if isinstance( self.location,FilePath):
            self.location = FilePath(path)
        else:
            self.location = path

    def get_name(self):
        if isinstance( self.location,FilePath):
            name = self.location.basename().decode("utf-8", "replace")
        else:
            name = self.location.decode("utf-8", "replace")
        return name

    def get_cover(self):
        try:
            return self.cover
        except:
            try:
                return self.parent.cover
            except:
                return ''

    def get_parent(self):
        return self.parent

    def get_item(self):
        return self.item

    def get_xml(self):
        return self.item.toString()

    def __repr__(self):
        return 'id: ' + str(self.id) + ' @ ' + self.get_name().encode('ascii','xmlcharrefreplace')
Beispiel #5
0
class FSItem(BackendItem):
    logCategory = 'fs_item'

    def __init__(self,
                 object_id,
                 parent,
                 path,
                 mimetype,
                 urlbase,
                 UPnPClass,
                 update=False,
                 store=None):
        BackendItem.__init__(self)
        self.id = object_id
        self.parent = parent
        if parent:
            parent.add_child(self, update=update)
        if mimetype == 'root':
            self.location = str(path)
        else:
            if mimetype == 'item' and path is None:
                path = os.path.join(parent.get_realpath(), str(self.id))
            #self.location = FilePath(unicode(path))
            self.location = FilePath(path)
        self.mimetype = mimetype
        if urlbase[-1] != '/':
            urlbase += '/'
        self.url = urlbase + str(self.id)

        self.store = store

        if parent == None:
            parent_id = -1
        else:
            parent_id = parent.get_id()

        self.item = UPnPClass(object_id, parent_id, self.get_name())
        if isinstance(self.item, Container):
            self.item.childCount = 0
        self.child_count = 0
        self.children = []
        self.sorted = False
        self.caption = None

        if mimetype in ['directory', 'root']:
            self.update_id = 0
            self.get_url = lambda: self.url
            self.get_path = lambda: None
            #self.item.searchable = True
            #self.item.searchClass = 'object'
            if (isinstance(self.location, FilePath)
                    and self.location.isdir() == True):
                self.check_for_cover_art()
                if getattr(self, 'cover', None):
                    _, ext = os.path.splitext(self.cover)
                    """ add the cover image extension to help clients not reacting on
                        the mimetype """
                    self.item.albumArtURI = ''.join(
                        (urlbase, str(self.id), '?cover', ext))
        else:
            self.get_url = lambda: self.url

            if self.mimetype.startswith('audio/'):
                if getattr(parent, 'cover', None):
                    _, ext = os.path.splitext(parent.cover)
                    """ add the cover image extension to help clients not reacting on
                        the mimetype """
                    self.item.albumArtURI = ''.join(
                        (urlbase, str(self.id), '?cover', ext))

            _, host_port, _, _, _ = urlsplit(urlbase)
            if host_port.find(':') != -1:
                host, port = tuple(host_port.split(':'))
            else:
                host = host_port

            try:
                size = self.location.getsize()
            except:
                size = 0

            if (self.store.server and self.store.server.coherence.config.get(
                    'transcoding', 'no') == 'yes'):
                if self.mimetype in ('application/ogg', 'audio/ogg',
                                     'audio/x-wav', 'audio/x-m4a',
                                     'application/x-flac'):
                    new_res = Resource(self.url + '/transcoded.mp3',
                                       'http-get:*:%s:*' % 'audio/mpeg')
                    new_res.size = None
                    #self.item.res.append(new_res)

            if mimetype != 'item':
                res = Resource('file://' + urllib.parse.quote(self.get_path()),
                               'internal:%s:%s:*' % (host, self.mimetype))
                res.size = size
                self.item.res.append(res)

            if mimetype != 'item':
                res = Resource(self.url, 'http-get:*:%s:*' % self.mimetype)
            else:
                res = Resource(self.url, 'http-get:*:*:*')

            res.size = size
            self.item.res.append(res)
            """ if this item is of type audio and we want to add a transcoding rule for it,
                this is the way to do it:

                create a new Resource object, at least a 'http-get'
                and maybe an 'internal' one too

                for transcoding to wav this looks like that

                res = Resource(url_for_transcoded audio,
                        'http-get:*:audio/x-wav:%s'% ';'.join(['DLNA.ORG_PN=JPEG_TN']+simple_dlna_tags))
                res.size = None
                self.item.res.append(res)
            """

            if (self.store.server and self.store.server.coherence.config.get(
                    'transcoding', 'no') == 'yes'):
                if self.mimetype in ('audio/mpeg', 'application/ogg',
                                     'audio/ogg', 'audio/x-wav', 'audio/x-m4a',
                                     'audio/flac', 'application/x-flac'):
                    dlna_pn = 'DLNA.ORG_PN=LPCM'
                    dlna_tags = simple_dlna_tags[:]
                    #dlna_tags[1] = 'DLNA.ORG_OP=00'
                    dlna_tags[2] = 'DLNA.ORG_CI=1'
                    new_res = Resource(
                        self.url + '?transcoded=lpcm', 'http-get:*:%s:%s' %
                        ('audio/L16;rate=44100;channels=2',
                         ';'.join([dlna_pn] + dlna_tags)))
                    new_res.size = None
                    #self.item.res.append(new_res)

                    if self.mimetype != 'audio/mpeg':
                        new_res = Resource(self.url + '?transcoded=mp3',
                                           'http-get:*:%s:*' % 'audio/mpeg')
                        new_res.size = None
                        #self.item.res.append(new_res)
            """ if this item is an image and we want to add a thumbnail for it
                we have to follow these rules:

                create a new Resource object, at least a 'http-get'
                and maybe an 'internal' one too

                for an JPG this looks like that

                res = Resource(url_for_thumbnail,
                        'http-get:*:image/jpg:%s'% ';'.join(['DLNA.ORG_PN=JPEG_TN']+simple_dlna_tags))
                res.size = size_of_thumbnail
                self.item.res.append(res)

                and for a PNG the Resource creation is like that

                res = Resource(url_for_thumbnail,
                        'http-get:*:image/png:%s'% ';'.join(simple_dlna_tags+['DLNA.ORG_PN=PNG_TN']))

                if not hasattr(self.item, 'attachments'):
                    self.item.attachments = {}
                self.item.attachments[key] = utils.StaticFile(filename_of_thumbnail)
            """

            if (self.mimetype in ('image/jpeg', 'image/png')
                    or self.mimetype.startswith('video/')):
                try:
                    filename, mimetype, dlna_pn = _find_thumbnail(
                        self.get_path())
                except NoThumbnailFound:
                    pass
                except:
                    self.warning(traceback.format_exc())
                else:
                    dlna_tags = simple_dlna_tags[:]
                    dlna_tags[
                        3] = 'DLNA.ORG_FLAGS=00f00000000000000000000000000000'

                    hash_from_path = str(id(filename))
                    new_res = Resource(
                        self.url + '?attachment=' + hash_from_path,
                        'http-get:*:%s:%s' %
                        (mimetype, ';'.join([dlna_pn] + dlna_tags)))
                    new_res.size = os.path.getsize(filename)
                    self.item.res.append(new_res)
                    if not hasattr(self.item, 'attachments'):
                        self.item.attachments = {}
                    self.item.attachments[hash_from_path] = utils.StaticFile(
                        filename)

            if self.mimetype.startswith('video/'):
                # check for a subtitles file
                caption, _ = os.path.splitext(self.get_path())
                caption = caption + '.srt'
                if os.path.exists(caption):
                    hash_from_path = str(id(caption))
                    mimetype = 'smi/caption'
                    new_res = Resource(
                        self.url + '?attachment=' + hash_from_path,
                        'http-get:*:%s:%s' % (mimetype, '*'))
                    new_res.size = os.path.getsize(caption)
                    self.caption = new_res.data
                    self.item.res.append(new_res)
                    if not hasattr(self.item, 'attachments'):
                        self.item.attachments = {}
                    self.item.attachments[hash_from_path] = utils.StaticFile(
                        caption)

            try:
                # FIXME: getmtime is deprecated in Twisted 2.6
                self.item.date = datetime.fromtimestamp(
                    self.location.getmtime())
            except:
                self.item.date = None

    def rebuild(self, urlbase):
        #print "rebuild", self.mimetype
        if self.mimetype != 'item':
            return
        #print "rebuild for", self.get_path()
        mimetype, _ = mimetypes.guess_type(self.get_path(), strict=False)
        if mimetype == None:
            return
        self.mimetype = mimetype
        #print "rebuild", self.mimetype
        UPnPClass = classChooser(self.mimetype)
        self.item = UPnPClass(self.id, self.parent.id, self.get_name())
        if getattr(self.parent, 'cover', None):
            _, ext = os.path.splitext(self.parent.cover)
            """ add the cover image extension to help clients not reacting on
                the mimetype """
            self.item.albumArtURI = ''.join(
                (urlbase, str(self.id), '?cover', ext))

        _, host_port, _, _, _ = urlsplit(urlbase)
        if host_port.find(':') != -1:
            host, port = tuple(host_port.split(':'))
        else:
            host = host_port

        res = Resource('file://' + urllib.parse.quote(self.get_path()),
                       'internal:%s:%s:*' % (host, self.mimetype))
        try:
            res.size = self.location.getsize()
        except:
            res.size = 0
        self.item.res.append(res)
        res = Resource(self.url, 'http-get:*:%s:*' % self.mimetype)

        try:
            res.size = self.location.getsize()
        except:
            res.size = 0
        self.item.res.append(res)

        try:
            # FIXME: getmtime is deprecated in Twisted 2.6
            self.item.date = datetime.fromtimestamp(self.location.getmtime())
        except:
            self.item.date = None

        self.parent.update_id += 1

    def check_for_cover_art(self):
        """ let's try to find in the current directory some jpg file,
            or png if the jpg search fails, and take the first one
            that comes around
        """
        try:
            jpgs = [
                i.path for i in self.location.children()
                if i.splitext()[1] in ('.jpg', '.JPG')
            ]
            try:
                self.cover = jpgs[0]
            except IndexError:
                pngs = [
                    i.path for i in self.location.children()
                    if i.splitext()[1] in ('.png', '.PNG')
                ]
                try:
                    self.cover = pngs[0]
                except IndexError:
                    return
        except UnicodeDecodeError:
            self.warning(
                "UnicodeDecodeError - there is something wrong with a file located in %r",
                self.location.path)

    def remove(self):
        #print "FSItem remove", self.id, self.get_name(), self.parent
        if self.parent:
            self.parent.remove_child(self)
        del self.item

    def add_child(self, child, update=False):
        self.children.append(child)
        self.child_count += 1
        if isinstance(self.item, Container):
            self.item.childCount += 1
        if update == True:
            self.update_id += 1
        self.sorted = False

    def remove_child(self, child):
        #print "remove_from %d (%s) child %d (%s)" % (self.id, self.get_name(), child.id, child.get_name())
        if child in self.children:
            self.child_count -= 1
            if isinstance(self.item, Container):
                self.item.childCount -= 1
            self.children.remove(child)
            self.update_id += 1
        self.sorted = False

    def get_children(self, start=0, request_count=0):
        if self.sorted == False:
            self.children.sort(key=_natural_key)
            self.sorted = True
        if request_count == 0:
            return self.children[start:]
        else:
            return self.children[start:request_count]

    def get_child_count(self):
        return self.child_count

    def get_id(self):
        return self.id

    def get_update_id(self):
        if hasattr(self, 'update_id'):
            return self.update_id
        else:
            return None

    def get_path(self):
        if isinstance(self.location, FilePath):
            return self.location.path
        else:
            self.location

    def get_realpath(self):
        if isinstance(self.location, FilePath):
            return self.location.path
        else:
            self.location

    def set_path(self, path=None, extension=None):
        if path is None:
            path = self.get_path()
        if extension is not None:
            path, old_ext = os.path.splitext(path)
            path = ''.join((path, extension))
        if isinstance(self.location, FilePath):
            self.location = FilePath(path)
        else:
            self.location = path

    def get_name(self):
        if isinstance(self.location, FilePath):
            name = self.location.basename().decode("utf-8", "replace")
        else:
            name = self.location.decode("utf-8", "replace")
        return name

    def get_cover(self):
        if self.cover:
            return self.cover
        try:
            return self.parent.cover
        except AttributeError:
            return None

    def get_parent(self):
        return self.parent

    def get_item(self):
        return self.item

    def get_xml(self):
        return self.item.toString()

    def __repr__(self):
        return 'id: ' + str(self.id) + ' @ ' + self.get_name().encode(
            'ascii', 'xmlcharrefreplace')