Beispiel #1
0
class Recording(BackendItem):

    logCategory = 'dvbd_store'

    def __init__(self,store,
                 id,parent_id,
                 file,title,
                 date,duration,
                 mimetype):

        self.store = store
        self.id = 'recording.%s' % id
        self.parent_id = parent_id
        self.real_id = id
    
        path = unicode(file)
        # make sure path is an absolute local path (and not an URL)
        if path.startswith("file://"):
        		path = path[7:]
        self.location = FilePath(path)

        self.title = unicode(title)
        self.mimetype = str(mimetype)
        self.date = datetime.fromtimestamp(int(date))
        self.duration = int(duration)
        try:
            self.size = self.location.getsize()
        except Exception, msg:
            self.size = 0
        self.bitrate = 0
        self.url = self.store.urlbase + str(self.id)
 def render_POST(self, request):
     
     context = self._context(request)
     
     form, status, appstruct = self._validate_form(request)
     
     if status == 'ok':
         filename = context['renderer'](appstruct)
         
         fileobj = FilePath(filename)
         
         request.setHeader('content-disposition', 'attachment; filename="output.pdf"')
         request.setHeader('content-length', str(fileobj.getsize()))
         request.setHeader('content-type', 'application/pdf')
         
         producer = NoRangeStaticProducer(request, fileobj.open('r'))
         
         producer.start()
         
         return NOT_DONE_YET
     
     data['renderer_form'] = form.render(appstruct)
     
     self.render_template(data, request)
     
     return NOT_DONE_YET
Beispiel #3
0
    def _get_new(self):
        """
        Get new requests to process. This will define the `main loop` of
        the FileChecker service.
        """

        # Check ACTIVE accounts to get files with running traffic captures
        active_requests = yield self._get_requests("ACTIVE")
        if active_requests:
            log.info("FILECHECKER:: Checking pcap files of ACTIVE accounts.")
            for request in active_requests:
                username = request[0]
                ip_addr = request[6]

                # Create path
                k = "{}_{}".format(username, ip_addr)
                pcap_file = "{}.pcap".format(k)
                pcap_file = os.path.join(self.pcaps_path, pcap_file)
                log.debug("FILECHECKER:: Checking {}".format(pcap_file))

                # First limit should be zero
                if self.reached_limits.get(k) is None:
                    self.reached_limits[k] = 0

                try:
                    fp = FilePath(pcap_file)
                    fsize = fp.getsize()
                    # Check each size limit only once
                    if self.reached_limits[k] + 1 < len(self.size_limit):
                        if fsize > int(
                                self.size_limit[self.reached_limits[k] + 1]):
                            text = "File {} has reached the limit of "\
                                   "{} bytes!".format(
                                pcap_file,
                                self.size_limit[self.reached_limits[k]+1]
                            )

                            log.info("FILECHECKER:: {}".format(text))
                            yield self.slackbot.post(text, self.slack_channel)

                            self.reached_limits[k] = self.reached_limits[k] + 1
                    else:
                        log.debug("FILECHECKER:: File {} has reached the last"
                                  " size limit of {} bytes!".format(
                                      pcap_file,
                                      self.size_limit[self.reached_limits[k]]))

                except Exception as e:
                    log.debug("FILECHECKER:: Could'nt get size: {}".format(e))

        else:
            log.debug("FILECHECKER:: No ACTIVE accounts - Keep waiting.")
Beispiel #4
0
 def get_index(self, path):
     index = {}
     for root, dirs, files in os.walk(path):
         index[root] = {}
         for f in files:
             path = os.path.join(root, f)
             filepath = FilePath(path)
             try:
                 index[path] = {
                     'mtime': filepath.getModificationTime(),
                     'size': filepath.getsize(),
                 }
             except OSError:
                 # file could be a broken symlink, or deleted mid-scan
                 continue
     return index
Beispiel #5
0
    def getDirectory(self, path='/'):

        self.fs = yield FilePath(path)

        if not self.fs.getPermissions():
            defer.returnValue(False)

        files = []

        for f in self.fs.listdir():
            if f == '/':
                continue

            fp = path+f
            fs = FilePath(fp)

            # dont follow symlinks
            if fs.realpath().path != fp:
                continue

            perm = None
            isdir = fs.isdir()
            size = fs.getsize()
            modified = datetime.utcfromtimestamp(fs.getModificationTime())

            df = DiscoveredFile(
                resource_id=self.data['resource_id'],
                file_path=path,
                file_name=f,
                file_isdir=isdir,
                file_size=size,
                file_modified=modified,
                file_perm=perm
            )

            print '[%s] LIST %s.' % (self.data['resource_name'], fp if not fp.endswith('.') else fp)
            files.append(df)

        defer.returnValue(files)
Beispiel #6
0
class Recording(BackendItem):
    logCategory = 'dvbd_store'

    def __init__(self, store, id, parent_id, file, title, date, duration,
                 mimetype):
        BackendItem.__init__(self)
        self.store = store
        self.id = 'recording.%s' % id
        self.parent_id = parent_id
        self.real_id = id

        path = str(file)
        # make sure path is an absolute local path (and not an URL)
        if path.startswith('file://'):
            path = path[7:]
        self.location = FilePath(path)

        self.title = str(title)
        self.mimetype = str(mimetype)
        self.date = datetime.fromtimestamp(int(date))
        self.duration = int(duration)
        try:
            self.size = self.location.getsize()
        except Exception as msg:
            self.size = 0
        self.bitrate = 0
        self.url = self.store.urlbase + str(self.id)

    def get_children(self, start=0, end=0):
        return []

    def get_child_count(self):
        return 0

    def get_item(self, parent_id=None):

        self.debug(f'Recording get_item {self.id} @ {self.parent_id}')

        # create item
        item = DIDLLite.VideoBroadcast(self.id, self.parent_id)
        item.date = self.date
        item.title = self.title

        # add http resource
        res = DIDLLite.Resource(self.url, f'http-get:*:{self.mimetype}:*')
        if self.size > 0:
            res.size = self.size
        if self.duration > 0:
            res.duration = str(self.duration)
        if self.bitrate > 0:
            res.bitrate = str(self.bitrate)
        item.res.append(res)

        # add internal resource
        res = DIDLLite.Resource(
            'file://' + urllib.parse.quote(self.get_path()),
            f'internal:{self.store.server.coherence.hostname}:'
            f'{self.mimetype}:*',
        )
        if self.size > 0:
            res.size = self.size
        if self.duration > 0:
            res.duration = str(self.duration)
        if self.bitrate > 0:
            res.bitrate = str(self.bitrate)
        item.res.append(res)

        return item

    def get_id(self):
        return self.id

    def get_name(self):
        return self.title

    def get_url(self):
        return self.url

    def get_path(self):
        return self.location.path
Beispiel #7
0
class Recording(BackendItem):

    logCategory = 'dvbd_store'

    def __init__(self,store,
                 id,parent_id,
                 file,title,
                 date,duration,
                 mimetype):

        self.store = store
        self.id = 'recording.%s' % id
        self.parent_id = parent_id
        self.real_id = id

        self.location = FilePath(unicode(file))
        self.title = unicode(title)
        self.mimetype = str(mimetype)
        self.date = datetime.fromtimestamp(int(date))
        self.duration = int(duration)
        self.size = self.location.getsize()
        self.bitrate = 0
        self.url = self.store.urlbase + str(self.id)

    def get_children(self, start=0, end=0):
        return []

    def get_child_count(self):
        return 0

    def get_item(self, parent_id=None):

        self.debug("Recording get_item %r @ %r" %(self.id,self.parent_id))

        # create item
        item = DIDLLite.VideoBroadcast(self.id,self.parent_id)
        item.date = self.date
        item.title = self.title

        # add http resource
        res = DIDLLite.Resource(self.url, 'http-get:*:%s:*' % self.mimetype)
        if self.size > 0:
            res.size = self.size
        if self.duration > 0:
            res.duration = str(self.duration)
        if self.bitrate > 0:
            res.bitrate = str(bitrate)
        item.res.append(res)

        # add internal resource
        res = DIDLLite.Resource('file://'+ urllib.quote(self.get_path()), 'internal:%s:%s:*' % (self.store.server.coherence.hostname,self.mimetype))
        if self.size > 0:
            res.size = self.size
        if self.duration > 0:
            res.duration = str(self.duration)
        if self.bitrate > 0:
            res.bitrate = str(bitrate)
        item.res.append(res)

        return item

    def get_id(self):
        return self.id

    def get_name(self):
        return self.title

    def get_url(self):
        return self.url

    def get_path(self):
        return self.location.path
Beispiel #8
0
class FSItem(BackendItem):
    logCategory = 'fs_item'

    def __init__(
        self,
        object_id,
        parent,
        path,
        mimetype,
        urlbase,
        UPnPClass,
        update=False,
        store=None,
    ):
        BackendItem.__init__(self)
        self.id = object_id
        self.parent = parent
        if parent:
            parent.add_child(self, update=update)
        if mimetype == 'root':
            self.location = str(path)
        else:
            if mimetype == 'item' and path is None:
                path = os.path.join(parent.get_realpath(), str(self.id))
            # self.location = FilePath(unicode(path))
            self.location = FilePath(path)
        self.mimetype = mimetype
        if urlbase[-1] != '/':
            urlbase += '/'
        self.url = urlbase + str(self.id)

        self.store = store

        if parent is None:
            parent_id = -1
        else:
            parent_id = parent.get_id()

        self.item = UPnPClass(object_id, parent_id, self.get_name())
        if isinstance(self.item, Container):
            self.item.childCount = 0
        self.child_count = 0
        self.children = []
        self.sorted = False
        self.caption = None

        if mimetype in ['directory', 'root']:
            self.update_id = 0
            self.get_url = lambda: self.url
            # self.item.searchable = True
            # self.item.searchClass = 'object'
            if (isinstance(self.location, FilePath)
                    and self.location.isdir() is True):
                self.check_for_cover_art()
                if getattr(self, 'cover', None):
                    _, ext = os.path.splitext(self.cover)
                    ''' add the cover image extension to help clients
                        not reacting on the mimetype '''
                    self.item.albumArtURI = ''.join(
                        (urlbase, str(self.id), '?cover', str(ext)))
        else:
            self.get_url = lambda: self.url

            if self.mimetype.startswith('audio/'):
                if getattr(parent, 'cover', None):
                    _, ext = os.path.splitext(parent.cover)
                    ''' add the cover image extension to help clients
                        not reacting on the mimetype '''
                    self.item.albumArtURI = ''.join(
                        (urlbase, str(self.id), '?cover', ext))

            _, host_port, _, _, _ = urlsplit(urlbase)
            if host_port.find(':') != -1:
                host, port = tuple(host_port.split(':'))
            else:
                host = host_port

            try:
                size = self.location.getsize()
            except Exception:
                size = 0

            if (self.store.server and self.store.server.coherence.config.get(
                    'transcoding', 'no') == 'yes'):
                if self.mimetype in (
                        'application/ogg',
                        'audio/ogg',
                        'audio/x-wav',
                        'audio/x-m4a',
                        'application/x-flac',
                ):
                    new_res = Resource(
                        self.url + '/transcoded.mp3',
                        f'http-get:*:{"audio/mpeg"}:*',
                    )
                    new_res.size = None
                    # self.item.res.append(new_res)

            if mimetype != 'item':
                res = Resource(
                    'file://' + quote(self.get_path(), encoding='utf-8'),
                    f'internal:{host}:{self.mimetype}:*',
                )
                res.size = size
                self.item.res.append(res)

            if mimetype != 'item':
                res = Resource(self.url, f'http-get:*:{self.mimetype}:*')
            else:
                res = Resource(self.url, 'http-get:*:*:*')

            res.size = size
            self.item.res.append(res)
            ''' if this item is of type audio and we want to add a transcoding
                rule for it, this is the way to do it:

                create a new Resource object, at least a 'http-get'
                and maybe an 'internal' one too

                for transcoding to wav this looks like that

                res = Resource(
                    url_for_transcoded audio,
                    'http-get:*:audio/x-wav:%s'% ';'.join(
                        ['DLNA.ORG_PN=JPEG_TN']+simple_dlna_tags))
                res.size = None
                self.item.res.append(res)
            '''

            if (self.store.server and self.store.server.coherence.config.get(
                    'transcoding', 'no') == 'yes'):
                if self.mimetype in (
                        'audio/mpeg',
                        'application/ogg',
                        'audio/ogg',
                        'audio/x-wav',
                        'audio/x-m4a',
                        'audio/flac',
                        'application/x-flac',
                ):
                    dlna_pn = 'DLNA.ORG_PN=LPCM'
                    dlna_tags = simple_dlna_tags[:]
                    # dlna_tags[1] = 'DLNA.ORG_OP=00'
                    dlna_tags[2] = 'DLNA.ORG_CI=1'
                    new_res = Resource(
                        self.url + '?transcoded=lpcm',
                        f'http-get:*:{"audio/L16;rate=44100;channels=2"}:'
                        f'{";".join([dlna_pn] + dlna_tags)}',
                    )
                    new_res.size = None
                    # self.item.res.append(new_res)

                    if self.mimetype != 'audio/mpeg':
                        new_res = Resource(
                            self.url + '?transcoded=mp3',
                            f'http-get:*:{"audio/mpeg"}:*',
                        )
                        new_res.size = None
                        # self.item.res.append(new_res)
            ''' if this item is an image and we want to add a thumbnail for it
                we have to follow these rules:

                create a new Resource object, at least a 'http-get'
                and maybe an 'internal' one too

                for an JPG this looks like that

                res = Resource(url_for_thumbnail,
                        'http-get:*:image/jpg:%s'% ';'.join(
                        ['DLNA.ORG_PN=JPEG_TN']+simple_dlna_tags))
                res.size = size_of_thumbnail
                self.item.res.append(res)

                and for a PNG the Resource creation is like that

                res = Resource(url_for_thumbnail,
                        'http-get:*:image/png:%s'% ';'.join(
                        simple_dlna_tags+['DLNA.ORG_PN=PNG_TN']))

                if not hasattr(self.item, 'attachments'):
                    self.item.attachments = {}
                self.item.attachments[key] = utils.StaticFile(
                filename_of_thumbnail)
            '''

            if (self.mimetype in ('image/jpeg', 'image/png')
                    or self.mimetype.startswith('video/')):
                try:
                    filename, mimetype, dlna_pn = _find_thumbnail(
                        self.get_path())
                except NoThumbnailFound:
                    pass
                except Exception:
                    self.warning(traceback.format_exc())
                else:
                    dlna_tags = simple_dlna_tags[:]
                    dlna_tags[
                        3] = 'DLNA.ORG_FLAGS=00f00000000000000000000000000000'

                    hash_from_path = str(id(filename))
                    new_res = Resource(
                        self.url + '?attachment=' + hash_from_path,
                        f'http-get:*:{mimetype}:'
                        f'{";".join([dlna_pn] + dlna_tags)}',
                    )
                    new_res.size = os.path.getsize(filename)
                    self.item.res.append(new_res)
                    if not hasattr(self.item, 'attachments'):
                        self.item.attachments = {}
                    self.item.attachments[hash_from_path] = utils.StaticFile(
                        filename)

            if self.mimetype.startswith('video/'):
                # check for a subtitles file
                caption, _ = os.path.splitext(self.get_path())
                caption = caption + '.srt'
                if os.path.exists(caption):
                    hash_from_path = str(id(caption))
                    mimetype = 'smi/caption'
                    new_res = Resource(
                        self.url + '?attachment=' + hash_from_path,
                        f'http-get:*:{mimetype}:{"*"}',
                    )
                    new_res.size = os.path.getsize(caption)
                    self.caption = new_res.data
                    self.item.res.append(new_res)
                    if not hasattr(self.item, 'attachments'):
                        self.item.attachments = {}
                    self.item.attachments[hash_from_path] = utils.StaticFile(
                        caption,
                        defaultType=mimetype,
                    )

            try:
                # FIXME: getmtime is deprecated in Twisted 2.6
                self.item.date = datetime.fromtimestamp(
                    self.location.getmtime())
            except Exception:
                self.item.date = None

    def rebuild(self, urlbase):
        # print('rebuild', self.mimetype)
        if self.mimetype != 'item':
            return
        # print('rebuild for', self.get_path())
        mimetype, _ = mimetypes.guess_type(self.get_path(), strict=False)
        if mimetype is None:
            return
        self.mimetype = mimetype
        # print('rebuild', self.mimetype)
        UPnPClass = classChooser(self.mimetype)
        self.item = UPnPClass(self.id, self.parent.id, self.get_name())
        if getattr(self.parent, 'cover', None):
            _, ext = os.path.splitext(self.parent.cover)
            # add the cover image extension to help
            # clients not reacting on the mimetype
            self.item.albumArtURI = ''.join(
                (urlbase, str(self.id), '?cover', ext))

        _, host_port, _, _, _ = urlsplit(urlbase)
        if host_port.find(':') != -1:
            host, port = tuple(host_port.split(':'))
        else:
            host = host_port

        res = Resource(
            'file://' + quote(self.get_path()),
            f'internal:{host}:{self.mimetype}:*',
        )
        try:
            res.size = self.location.getsize()
        except Exception:
            res.size = 0
        self.item.res.append(res)
        res = Resource(self.url, f'http-get:*:{self.mimetype}:*')

        try:
            res.size = self.location.getsize()
        except Exception:
            res.size = 0
        self.item.res.append(res)

        try:
            # FIXME: getmtime is deprecated in Twisted 2.6
            self.item.date = datetime.fromtimestamp(self.location.getmtime())
        except Exception:
            self.item.date = None

        self.parent.update_id += 1

    def check_for_cover_art(self):
        ''' let's try to find in the current directory some jpg file,
            or png if the jpg search fails, and take the first one
            that comes around
        '''
        try:
            jpgs = [
                i.path for i in self.location.children()
                if i.splitext()[1] in ('.jpg', '.JPG')
            ]
            try:
                self.cover = jpgs[0]
            except IndexError:
                pngs = [
                    i.path for i in self.location.children()
                    if i.splitext()[1] in ('.png', '.PNG')
                ]
                try:
                    self.cover = pngs[0]
                except IndexError:
                    return
        except UnicodeDecodeError:
            self.warning(
                f'UnicodeDecodeError - there is something wrong with a ' +
                f'file located in {self.location.path}')

    def remove(self):
        # print('FSItem remove', self.id, self.get_name(), self.parent)
        if self.parent:
            self.parent.remove_child(self)
        del self.item

    def add_child(self, child, update=False):
        self.children.append(child)
        self.child_count += 1
        if isinstance(self.item, Container):
            self.item.childCount += 1
        if update:
            self.update_id += 1
        self.sorted = False

    def remove_child(self, child):
        # print(f'remove_from {self.id:d} ({self.get_name()}) '
        #       f'child {child.id:d} ({child.get_name()})')
        if child in self.children:
            self.child_count -= 1
            if isinstance(self.item, Container):
                self.item.childCount -= 1
            self.children.remove(child)
            self.update_id += 1
        self.sorted = False

    def get_children(self, start=0, request_count=0):
        if not self.sorted:
            self.children.sort(key=_natural_key)
            self.sorted = True
        if request_count == 0:
            return self.children[start:]
        else:
            return self.children[start:request_count]

    def get_child_count(self):
        return self.child_count

    def get_id(self):
        return self.id

    def get_update_id(self):
        if hasattr(self, 'update_id'):
            return self.update_id
        else:
            return None

    def get_path(self):
        if self.mimetype in ['directory', 'root']:
            return None
        if isinstance(self.location, FilePath):
            return self.location.path
        else:
            return self.location

    def get_realpath(self):
        if isinstance(self.location, FilePath):
            return self.location.path
        else:
            return self.location

    def set_path(self, path=None, extension=None):
        if path is None:
            path = self.get_path()
        if extension is not None:
            path, old_ext = os.path.splitext(path)
            path = ''.join((path, extension))
        if isinstance(self.location, FilePath):
            self.location = FilePath(path)
        else:
            self.location = path

    def get_name(self):
        if isinstance(self.location, FilePath):
            name = self.location.basename()
        else:
            name = self.location
        return name

    def get_cover(self):
        if self.cover:
            return self.cover
        try:
            return self.parent.cover
        except AttributeError:
            return None

    def get_parent(self):
        return self.parent

    def get_item(self):
        return self.item

    def get_xml(self):
        return self.item.toString()

    def __repr__(self):
        return ('id: ' + str(self.id) + ' @ ' +
                str(self.get_name().encode('ascii', 'xmlcharrefreplace')))
Beispiel #9
0
class FSItem(BackendItem):
    logCategory = 'fs_item'

    def __init__(self, object_id, parent, path, mimetype, urlbase, UPnPClass,update=False):
        self.id = object_id
        self.parent = parent
        if parent:
            parent.add_child(self,update=update)
        if mimetype == 'root':
            self.location = unicode(path)
        else:
            if mimetype == 'item' and path is None:
                path = os.path.join(parent.get_path(),unicode(self.id))
            #self.location = FilePath(unicode(path))
            self.location = FilePath(path)
        self.mimetype = mimetype
        if urlbase[-1] != '/':
            urlbase += '/'
        self.url = urlbase + str(self.id)


        if parent == None:
            parent_id = -1
        else:
            parent_id = parent.get_id()

        self.item = UPnPClass(object_id, parent_id, self.get_name())
        if isinstance(self.item, Container):
            self.item.childCount = 0
        self.child_count = 0
        self.children = []


        if mimetype in ['directory','root']:
            self.update_id = 0
            self.get_url = lambda : self.url
            self.get_path = lambda : None
            #self.item.searchable = True
            #self.item.searchClass = 'object'
            if(isinstance(self.location,FilePath) and
               self.location.isdir() == True):
                self.check_for_cover_art()
                if hasattr(self, 'cover'):
                    _,ext =  os.path.splitext(self.cover)
                    """ add the cover image extension to help clients not reacting on
                        the mimetype """
                    self.item.albumArtURI = ''.join((urlbase,str(self.id),'?cover',ext))
        else:
            self.get_url = lambda : self.url

            if self.mimetype.startswith('audio/'):
                if hasattr(parent, 'cover'):
                    _,ext =  os.path.splitext(parent.cover)
                    """ add the cover image extension to help clients not reacting on
                        the mimetype """
                    self.item.albumArtURI = ''.join((urlbase,str(self.id),'?cover',ext))

            _,host_port,_,_,_ = urlsplit(urlbase)
            if host_port.find(':') != -1:
                host,port = tuple(host_port.split(':'))
            else:
                host = host_port

            try:
                size = self.location.getsize()
            except:
                size = 0

            if mimetype != 'item':
                res = Resource('file://'+ urllib.quote(self.get_path()), 'internal:%s:%s:*' % (host,self.mimetype))
                res.size = size
                self.item.res.append(res)

            if mimetype != 'item':
                res = Resource(self.url, 'http-get:*:%s:*' % self.mimetype)
            else:
                res = Resource(self.url, 'http-get:*:*:*')

            res.size = size
            self.item.res.append(res)


            """ if this item is an image and we want to add a thumbnail for it
                we have to follow these rules:

                create a new Resource object, at least a 'http-get'
                and maybe an 'internal' one too

                for an JPG this looks like that

                res = Resource(url_for_thumbnail,
                        'http-get:*:image/jpg:%s'% ';'.join(simple_dlna_tags+('DLNA.ORG_PN=JPEG_TN',)))
                res.size = size_of_thumbnail
                self.item.res.append(res)

                and for a PNG the Resource creation is like that

                res = Resource(url_for_thumbnail,
                        'http-get:*:image/png:%s'% ';'.join(simple_dlna_tags+('DLNA.ORG_PN=PNG_TN',)))

                if not hasattr(self.item, 'attachments'):
                    self.item.attachments = {}
                self.item.attachments[key] = utils.StaticFile(filename_of_thumbnail)
            """

            if self.mimetype in ('image/jpeg', 'image/png'):
                path = self.get_path()
                thumbnail = os.path.join(os.path.dirname(path),'.thumbs',os.path.basename(path))
                if os.path.exists(thumbnail):
                    mimetype,_ = mimetypes.guess_type(thumbnail, strict=False)
                    if mimetype in ('image/jpeg','image/png'):
                        if mimetype == 'image/jpeg':
                            dlna_pn = 'DLNA.ORG_PN=JPEG_TN'
                        else:
                            dlna_pn = 'DLNA.ORG_PN=PNG_TN'

                        hash_from_path = str(id(thumbnail))
                        new_res = Resource(self.url+'?attachment='+hash_from_path,
                            'http-get:*:%s:%s' % (mimetype, ';'.join(simple_dlna_tags+(dlna_pn,))))
                        new_res.size = os.path.getsize(thumbnail)
                        self.item.res.append(new_res)
                        if not hasattr(self.item, 'attachments'):
                            self.item.attachments = {}
                        self.item.attachments[hash_from_path] = utils.StaticFile(urllib.quote(thumbnail))


            try:
                # FIXME: getmtime is deprecated in Twisted 2.6
                self.item.date = datetime.fromtimestamp(self.location.getmtime())
            except:
                self.item.date = None

    def rebuild(self, urlbase):
        #print "rebuild", self.mimetype
        if self.mimetype != 'item':
            return
        #print "rebuild for", self.get_path()
        mimetype,_ = mimetypes.guess_type(self.get_path(),strict=False)
        if mimetype == None:
            return
        self.mimetype = mimetype
        #print "rebuild", self.mimetype
        UPnPClass = classChooser(self.mimetype)
        self.item = UPnPClass(self.id, self.parent.id, self.get_name())
        if hasattr(self.parent, 'cover'):
            _,ext =  os.path.splitext(self.parent.cover)
            """ add the cover image extension to help clients not reacting on
                the mimetype """
            self.item.albumArtURI = ''.join((urlbase,str(self.id),'?cover',ext))

        _,host_port,_,_,_ = urlsplit(urlbase)
        if host_port.find(':') != -1:
            host,port = tuple(host_port.split(':'))
        else:
            host = host_port

        res = Resource('file://'+urllib.quote(self.get_path()), 'internal:%s:%s:*' % (host,self.mimetype))
        try:
            res.size = self.location.getsize()
        except:
            res.size = 0
        self.item.res.append(res)
        res = Resource(self.url, 'http-get:*:%s:*' % self.mimetype)

        try:
            res.size = self.location.getsize()
        except:
            res.size = 0
        self.item.res.append(res)

        try:
            # FIXME: getmtime is deprecated in Twisted 2.6
            self.item.date = datetime.fromtimestamp(self.location.getmtime())
        except:
            self.item.date = None

        self.parent.update_id += 1

    def check_for_cover_art(self):
        """ let's try to find in the current directory some jpg file,
            or png if the jpg search fails, and take the first one
            that comes around
        """
        try:
            jpgs = [i.path for i in self.location.children() if i.splitext()[1] in ('.jpg', '.JPG')]
            try:
                self.cover = jpgs[0]
            except IndexError:
                pngs = [i.path for i in self.location.children() if i.splitext()[1] in ('.png', '.PNG')]
                try:
                    self.cover = pngs[0]
                except IndexError:
                    return
        except UnicodeDecodeError:
            self.warning("UnicodeDecodeError - there is something wrong with a file located in %r", self.location.path)

    def remove(self):
        #print "FSItem remove", self.id, self.get_name(), self.parent
        if self.parent:
            self.parent.remove_child(self)
        del self.item

    def add_child(self, child, update=False):
        self.children.append(child)
        self.child_count += 1
        if isinstance(self.item, Container):
            self.item.childCount += 1
        if update == True:
            self.update_id += 1

    def remove_child(self, child):
        #print "remove_from %d (%s) child %d (%s)" % (self.id, self.get_name(), child.id, child.get_name())
        if child in self.children:
            self.child_count -= 1
            if isinstance(self.item, Container):
                self.item.childCount -= 1
            self.children.remove(child)
            self.update_id += 1

    def get_children(self,start=0,request_count=0):
        if request_count == 0:
            return self.children[start:]
        else:
            return self.children[start:request_count]

    def get_child_count(self):
        return self.child_count

    def get_id(self):
        return self.id

    def get_update_id(self):
        if hasattr(self, 'update_id'):
            return self.update_id
        else:
            return None

    def get_path(self):
        if isinstance( self.location,FilePath):
            return self.location.path
        else:
            self.location

    def set_path(self,path=None,extension=None):
        if path is None:
            path = self.get_path()
        if extension is not None:
            path,old_ext = os.path.splitext(path)
            path = ''.join((path,extension))
        if isinstance( self.location,FilePath):
            self.location = FilePath(path)
        else:
            self.location = path

    def get_name(self):
        if isinstance( self.location,FilePath):
            name = self.location.basename().decode("utf-8", "replace")
        else:
            name = self.location.decode("utf-8", "replace")
        return name

    def get_cover(self):
        try:
            return self.cover
        except:
            try:
                return self.parent.cover
            except:
                return ''

    def get_parent(self):
        return self.parent

    def get_item(self):
        return self.item

    def get_xml(self):
        return self.item.toString()

    def __repr__(self):
        return 'id: ' + str(self.id) + ' @ ' + self.get_name().encode('ascii','xmlcharrefreplace')
Beispiel #10
0
    def startCopying(self, msg):
        def _done(ign, src, dst, f):
            log.msg("File copied successfully %(src)s -> %(dst)s" % {
                'src': src,
                'dst': dst
            })
            #            self.publish('file-transfer-ack', fileTransferReq, id=msg.id.getValue(), state=0, table=msg.table.getValue())
            self.amqp.send_message(exchange="rb",
                                   routing_key="file-transfer-ack",
                                   msg=json.dumps({
                                       'id': msg['id'],
                                       'state': 0,
                                       'table': msg['table'],
                                   }))

        def _err_done(reason, src, dst, sendreq=None):
            log.msg(
                "File transfer error %(src)s -> %(dst)s. Reason: %(reason)s" %
                {
                    'src': src,
                    'dst': dst,
                    'reason': reason.getErrorMessage()
                },
                logLevel=logging.ERROR)
            if sendreq:
                #self.publish('file-transfer-ack', fileTransferReq, id=msg.id.getValue(), state=1, table=msg.table.getValue())
                self.amqp.send_message(exchange="rb",
                                       routing_key="file-transfer-ack",
                                       msg=json.dumps({
                                           'id': msg['id'],
                                           'state': 1,
                                           'table': msg['table'],
                                       }))
            return defer.fail(reason)

        if self.working > self.QUEUESIZE:
            self.queue.put(msg)

            return

        self.incpendingqueue(None, msg=msg)

        srcfile = msg['src']
        dstfile = msg['dst']
        s = FilePath(srcfile)
        d = FilePath(dstfile)
        if s.exists() and d.exists():
            if s.getsize() == d.getsize():
                log.msg("Ignore coping, files have the same size: %s -> %s" %
                        (srcfile, dstfile))
                self.decpendingqueue(None, msg=msg)
                #self.publish('file-transfer-ack', fileTransferReq, id=msg.id.getValue(), state=0, table=msg.table.getValue())
                self.amqp.send_message(exchange="rb",
                                       routing_key="file-transfer-ack",
                                       msg=json.dumps({
                                           'id': msg['id'],
                                           'state': 0,
                                           'table': msg['table'],
                                       }))

                return
        src = FileSender()
        try:
            f = open(srcfile, 'r')

        except Exception, e:
            log.msg(str(e), logLevel=logging.ERROR)
            #                f=None
            return _err_done(Failure(e), srcfile, dstfile,
                             'err').addBoth(self.decpendingqueue, msg=msg)
Beispiel #11
0
    def startCopying(self, msg):
        def _done(ign, src, dst, f):
            log.msg("File copied successfully %(src)s -> %(dst)s" % {
                'src': src,
                'dst': dst
            })
            self.publish('file-transfer-ack',
                         fileTransferReq,
                         id=msg.id.getValue(),
                         state=0,
                         table=msg.table.getValue())

        def _err_done(reason, src, dst, sendreq=None):
            log.msg(
                "File transfer error %(src)s -> %(dst)s. Reason: %(reason)s" %
                {
                    'src': src,
                    'dst': dst,
                    'reason': reason.getErrorMessage()
                },
                logLevel=logging.ERROR)
            if sendreq:
                self.publish('file-transfer-ack',
                             fileTransferReq,
                             id=msg.id.getValue(),
                             state=1,
                             table=msg.table.getValue())
            return defer.fail(reason)

        if self.working > self.QUEUESIZE:
            self.queue.put(msg)

            return

        self.incpendingqueue(None, msg=msg)

        srcfile = msg.src.getValue()
        dstfile = msg.dst.getValue()
        s = FilePath(srcfile)
        d = FilePath(dstfile)
        if s.exists() and d.exists():
            if s.getsize() == d.getsize():
                log.msg("Ignore coping, files have the same size: %s -> %s" %
                        (srcfile, dstfile))
                self.decpendingqueue("SAME_SIZE", msg=msg)
                self.publish('file-transfer-ack',
                             fileTransferReq,
                             id=msg.id.getValue(),
                             state=0,
                             table=msg.table.getValue())
                return
        src = FileSender()

        if self.conf.get('transport', 'CHUNK_SIZE_POWER'):
            src.CHUNK_SIZE = 2**int(
                self.conf.get('transport', 'CHUNK_SIZE_POWER'))

        try:
            f = open(srcfile, 'r')

        except Exception, e:
            log.msg(str(e), logLevel=logging.ERROR)
            #                f=None
            return _err_done(Failure(e), srcfile, dstfile,
                             'err').addBoth(self.decpendingqueue, msg=msg)
Beispiel #12
0
import os
import re
import sys

from datetime import datetime, timedelta
from twisted.python.filepath import FilePath

date = datetime.now() - timedelta(days=1)
datestr = date.strftime("%Y-%m-%d")

os.system("wget -c http://eve-central.com/dumps/%s.dump.gz" % (datestr,))
os.system("gunzip %s.dump.gz" % (datestr,))

transactionsFilePath = FilePath("%s.dump" % (datestr,))
#transactionsFilePath = FilePath("transactions.csv")
size = float(transactionsFilePath.getsize())
fp = transactionsFilePath.open("r")

db = MySQLdb.connect("localhost", "root", db="frisky-iskies")
c = db.cursor()

c.execute("truncate table transactions")
c.execute("alter table transactions disable keys")

durationRe = re.compile(r"\D.*")

with open("transactions.csv") as csvfile:
    csvreader = csv.reader(csvfile)
    csvreader.next()
    i = 0
    for row in csvreader: