Esempio n. 1
0
    def getMeta(self, uri, depth=0, list=None, **kwargs):
        """
        Gets the live + dead properties of an object
        """        
        uris = [uri]
        if depth > 0 and self.isCollection(uri, **kwargs):
            resources = os.listdir(self._translateUri(uri, **kwargs))
            prefix = uri
            if uri != '/':
                prefix += '/'
            for resource in resources:
                uris.append(prefix + resource)

        meta = {}

        for href in uris:
            path = self._translateUri(href, **kwargs)

            meta[href] = self.propertyProvider.get(path)

            if meta[href].has_key('{DAV:}resourcetype'):
                if meta[href]['{DAV:}resourcetype'] == '':
                    meta[href]['{DAV:}resourcetype'] = 0
                else:
                    meta[href]['{DAV:}resourcetype'] = int(meta[href]['{DAV:}resourcetype'])

            props = {}
            if not meta[href].has_key('{DAV:}getcontenttype'):
                mime_type, encoding = utils.getMime(path)
                props.update({'{DAV:}getcontenttype': mime_type})
                if encoding is not None:
                    props.update({'{DAV:}encoding': encoding})

            if not meta[href].has_key('{DAV:}getcontentlength'):
                stats = os.stat(path)
                dir   = stat.S_ISDIR(stats.st_mode)
                if dir:
                    dir = 1
                    mime_type = 'application/x-directory'
                else:
                    dir = 0
                    
                props.update({
                    '{DAV:}getlastmodified': stats.st_mtime,
                    '{DAV:}getcontentlength': stats.st_size,
                    '{DAV:}creationdate': stats.st_ctime,
                    '{DAV:}resourcetype': dir
                })

            meta[href].update(props)
            for field, value in props.items():
                self.propertyProvider.set(path, field, value)

        return meta
Esempio n. 2
0
    def create(self, uri, data, env=None, expectedSize=None, **kwargs):
        localPath = self._translateUri(uri, **kwargs)
        
        self._saveToFile(data,localPath, expectedSize)

        mime_type, encoding = utils.getMime(localPath)
        stats = os.stat(localPath)
        return {
                  '{DAV:}getcontenttype': mime_type,
                  '{DAV:}encoding': encoding,
                  '{DAV:}getcontentlength': stats.st_size,
                  '{DAV:}getlastmodified' : stats.st_mtime,
                }
Esempio n. 3
0
    def process(self, user, request, resource, path, meta):
        # the smalles JPEG image is 134 bytes
        # @see: http://stackoverflow.com/questions/2253404/what-is-the-smallest-valid-jpeg-file-size-in-bytes

        if not meta:
            meta = resource.storageProvider.getMeta(path, user=user)
            meta = meta[path]

        # don't bother the conversion queue for items that are smaller than minSize bytes
        size = None
        try:
            size = int(meta['{DAV:}getcontentlength'])
            if size < conf.server.Config['jobclient']['minSize']:
                return
        except Exception as ex:
            logging.getLogger().warn("Incorrect job client configuration: %s" % ex)

        # Step 0: Deduplicate the data, if the storage provider supports it
        minDeduplicationSize = 0
        try:
            minDeduplicationSize = conf.server.Config['jobclient']['minDedupSize']
        except Exception as ex:
            logging.getLogger().warn("Incorrect job client configuration: %s" % ex)

        if size and size > minDeduplicationSize:
            try:
                deduplicate = getattr(resource.storageProvider, 'deduplicate')
                if deduplicate(path, user=user):
                    try:
                        addCache = getattr(resource.storageProvider, 'addCache')
                        # if we use cache decorator -> then refresh the cache for this item
                        addCache(path, user=user)
                    except:
                        pass
                    # This is already processed item -> do not run the conversion for it.
                    return
            except Exception as ex:
                logging.getLogger().warn('Deduplicaiton failed: %s, Exception: %s' % (ex, traceback.format_exc()))
        
        connection = ConnectionPool().get().getConnection()
                               
        channel = connection.channel()
        channel.basic_qos(0,1,False)
        """
        Before we start figure out the exact mime type of the file
        and some additional properties
        """
        localPath = resource.storageProvider._translateUri(path, user=user)

        # Step 1: Figure out the mime type
        mediaInfo = utils.mediaInfo(localPath)

        mimeType = None
        if mediaInfo.has_key('General') and mediaInfo['General'].has_key('InternetMediaType'):
            mimeType = mediaInfo['General']['InternetMediaType']
            resource.storageProvider.setMeta(path, {'{DAV:}getcontenttype': mimeType}, user=user)
        else:
            [mimeType, encoding] = utils.getMime(localPath)

        if not mimeType:
            return
        
        # Step 2: For the current user, his/hers conversion group,  path and mime-major get the conversions
        #         that can be applied
        userId = user.get('id')
        dir    =  os.path.dirname(path)
        conversionGroup = user.get('caps:conversion_group') # the convertion group is the default convertions that the user has
        (major, minor) = mimeType.split('/',1)

        converter   = Conversions()
        allConversions = { '*': [], major: []}
        if conversionGroup:
            groupAllConversions = converter.findRange(
                            "%s-%s-%s" % (conversionGroup, '*', '/'),
                            "%s-%s-%s" % (conversionGroup, '*',dir),
                              )

            groupMimeConversions = converter.findRange(
                            "%s-%s-%s" % (conversionGroup, major, '/'),
                            "%s-%s-%s" % (conversionGroup, major, dir),
                              )

            allConversions = {
                '*'   : groupAllConversions,
                major : groupMimeConversions
            }
        
        
        userAllConversions = converter.findRange(
                            "%s-%s-%s" % (userId, '*', '/'),
                            "%s-%s-%s" % (userId, '*', dir),
                              )

        userMimeConversions = converter.findRange(
                            "%s-%s-%s" % (userId, major, '/'),
                            "%s-%s-%s" % (userId, major, dir),
                              )
       
        allConversions['*']   += userAllConversions
        allConversions[major] += userMimeConversions

        cipher = coder.SecureLink(**conf.server.Config['secret'])
        resourceUri = path.replace(request.path,request.uri)
        tokenServer = 'http://'+TokenVHost.host[0]+':'+TokenVHost.port+'/'
        tokenUrl = tokenServer + urllib.quote(cipher.encode(['GET','PROPPATCH'], user.getIdentity(), resourceUri))

        [viewLocalFolder, viewBaseURL] = resource.storageProvider.getView(path, request, user=user)
        # create the local base folder where the data will be stored
        if viewLocalFolder and not os.path.exists(viewLocalFolder):
            logging.getLogger().debug("Create new view Local Folder: %s" % viewLocalFolder)
            os.makedirs(viewLocalFolder)

        jobs = {}
        for key, convertions in allConversions.items():
            for conversion in convertions:
                actions = {}
                for (name, value) in conversion.items():
                    try:
                        if name.find('action:') != -1:
                            action = name[len('action:'):]

                            accepted = conversion.get('accept:'+action)
                            if accepted:
                                accepted = json.loads(accepted)
                                if not minor in accepted:
                                    continue

                            actions[action] = json.loads(value)
                    except:
                        logging.getLogger().error('Conversion data is wrong: %s' % traceback.format_exc())
                        continue

                for name, options in actions.items():
                    routingKey = "%s.%s" % (key, name)
                    if not jobs.has_key(routingKey):
                        jobs[routingKey] = {
                            'url': tokenUrl,
                            'source': localPath,
                            'meta' : meta,
                            'mime' : mimeType,
                            'targetFolder': viewLocalFolder,
                            'targetBaseURL': viewBaseURL,
                            'options': {}
                        }

                    jobs[routingKey]['options'].update(options)



        for routingKey, job in jobs.items():
            logging.getLogger().debug("Sending: RoutingKey:[%s], Message:[%s]" % (routingKey, json.dumps(job)))
            msg = amqp.Message(json.dumps(job))
            msg.properties["delivery_mode"] = 2
            channel.basic_publish(msg,
                    exchange='workers',
                    routing_key=routingKey
            )

        channel.close()
Esempio n. 4
0
    def process(self, user, request, resource, path, meta):
        # the smalles JPEG image is 134 bytes
        # @see: http://stackoverflow.com/questions/2253404/what-is-the-smallest-valid-jpeg-file-size-in-bytes

        if not meta:
            meta = resource.storageProvider.getMeta(path, user=user)
            meta = meta[path]

        # don't bother the conversion queue for items that are smaller than minSize bytes
        size = None
        try:
            size = int(meta['{DAV:}getcontentlength'])
            if size < conf.server.Config['jobclient']['minSize']:
                return
        except Exception as ex:
            logging.getLogger().warn("Incorrect job client configuration: %s" %
                                     ex)

        # Step 0: Deduplicate the data, if the storage provider supports it
        minDeduplicationSize = 0
        try:
            minDeduplicationSize = conf.server.Config['jobclient'][
                'minDedupSize']
        except Exception as ex:
            logging.getLogger().warn("Incorrect job client configuration: %s" %
                                     ex)

        if size and size > minDeduplicationSize:
            try:
                deduplicate = getattr(resource.storageProvider, 'deduplicate')
                if deduplicate(path, user=user):
                    try:
                        addCache = getattr(resource.storageProvider,
                                           'addCache')
                        # if we use cache decorator -> then refresh the cache for this item
                        addCache(path, user=user)
                    except:
                        pass
                    # This is already processed item -> do not run the conversion for it.
                    return
            except Exception as ex:
                logging.getLogger().warn(
                    'Deduplicaiton failed: %s, Exception: %s' %
                    (ex, traceback.format_exc()))

        connection = ConnectionPool().get().getConnection()

        channel = connection.channel()
        channel.basic_qos(0, 1, False)
        """
        Before we start figure out the exact mime type of the file
        and some additional properties
        """
        localPath = resource.storageProvider._translateUri(path, user=user)

        # Step 1: Figure out the mime type
        mediaInfo = utils.mediaInfo(localPath)

        mimeType = None
        if mediaInfo.has_key('General') and mediaInfo['General'].has_key(
                'InternetMediaType'):
            mimeType = mediaInfo['General']['InternetMediaType']
            resource.storageProvider.setMeta(
                path, {'{DAV:}getcontenttype': mimeType}, user=user)
        else:
            [mimeType, encoding] = utils.getMime(localPath)

        if not mimeType:
            return

        # Step 2: For the current user, his/hers conversion group,  path and mime-major get the conversions
        #         that can be applied
        userId = user.get('id')
        dir = os.path.dirname(path)
        conversionGroup = user.get(
            'caps:conversion_group'
        )  # the convertion group is the default convertions that the user has
        (major, minor) = mimeType.split('/', 1)

        converter = Conversions()
        allConversions = {'*': [], major: []}
        if conversionGroup:
            groupAllConversions = converter.findRange(
                "%s-%s-%s" % (conversionGroup, '*', '/'),
                "%s-%s-%s" % (conversionGroup, '*', dir),
            )

            groupMimeConversions = converter.findRange(
                "%s-%s-%s" % (conversionGroup, major, '/'),
                "%s-%s-%s" % (conversionGroup, major, dir),
            )

            allConversions = {
                '*': groupAllConversions,
                major: groupMimeConversions
            }

        userAllConversions = converter.findRange(
            "%s-%s-%s" % (userId, '*', '/'),
            "%s-%s-%s" % (userId, '*', dir),
        )

        userMimeConversions = converter.findRange(
            "%s-%s-%s" % (userId, major, '/'),
            "%s-%s-%s" % (userId, major, dir),
        )

        allConversions['*'] += userAllConversions
        allConversions[major] += userMimeConversions

        cipher = coder.SecureLink(**conf.server.Config['secret'])
        resourceUri = path.replace(request.path, request.uri)
        tokenServer = 'http://' + TokenVHost.host[
            0] + ':' + TokenVHost.port + '/'
        tokenUrl = tokenServer + urllib.quote(
            cipher.encode(['GET', 'PROPPATCH'], user.getIdentity(),
                          resourceUri))

        [viewLocalFolder,
         viewBaseURL] = resource.storageProvider.getView(path,
                                                         request,
                                                         user=user)
        # create the local base folder where the data will be stored
        if viewLocalFolder and not os.path.exists(viewLocalFolder):
            logging.getLogger().debug("Create new view Local Folder: %s" %
                                      viewLocalFolder)
            os.makedirs(viewLocalFolder)

        jobs = {}
        for key, convertions in allConversions.items():
            for conversion in convertions:
                actions = {}
                for (name, value) in conversion.items():
                    try:
                        if name.find('action:') != -1:
                            action = name[len('action:'):]

                            accepted = conversion.get('accept:' + action)
                            if accepted:
                                accepted = json.loads(accepted)
                                if not minor in accepted:
                                    continue

                            actions[action] = json.loads(value)
                    except:
                        logging.getLogger().error(
                            'Conversion data is wrong: %s' %
                            traceback.format_exc())
                        continue

                for name, options in actions.items():
                    routingKey = "%s.%s" % (key, name)
                    if not jobs.has_key(routingKey):
                        jobs[routingKey] = {
                            'url': tokenUrl,
                            'source': localPath,
                            'meta': meta,
                            'mime': mimeType,
                            'targetFolder': viewLocalFolder,
                            'targetBaseURL': viewBaseURL,
                            'options': {}
                        }

                    jobs[routingKey]['options'].update(options)

        for routingKey, job in jobs.items():
            logging.getLogger().debug(
                "Sending: RoutingKey:[%s], Message:[%s]" %
                (routingKey, json.dumps(job)))
            msg = amqp.Message(json.dumps(job))
            msg.properties["delivery_mode"] = 2
            channel.basic_publish(msg,
                                  exchange='workers',
                                  routing_key=routingKey)

        channel.close()