def getMeta(self, uri, depth=0, list=None, **kwargs): """ Gets the live + dead properties of an object """ uris = [uri] if depth > 0 and self.isCollection(uri, **kwargs): resources = os.listdir(self._translateUri(uri, **kwargs)) prefix = uri if uri != '/': prefix += '/' for resource in resources: uris.append(prefix + resource) meta = {} for href in uris: path = self._translateUri(href, **kwargs) meta[href] = self.propertyProvider.get(path) if meta[href].has_key('{DAV:}resourcetype'): if meta[href]['{DAV:}resourcetype'] == '': meta[href]['{DAV:}resourcetype'] = 0 else: meta[href]['{DAV:}resourcetype'] = int(meta[href]['{DAV:}resourcetype']) props = {} if not meta[href].has_key('{DAV:}getcontenttype'): mime_type, encoding = utils.getMime(path) props.update({'{DAV:}getcontenttype': mime_type}) if encoding is not None: props.update({'{DAV:}encoding': encoding}) if not meta[href].has_key('{DAV:}getcontentlength'): stats = os.stat(path) dir = stat.S_ISDIR(stats.st_mode) if dir: dir = 1 mime_type = 'application/x-directory' else: dir = 0 props.update({ '{DAV:}getlastmodified': stats.st_mtime, '{DAV:}getcontentlength': stats.st_size, '{DAV:}creationdate': stats.st_ctime, '{DAV:}resourcetype': dir }) meta[href].update(props) for field, value in props.items(): self.propertyProvider.set(path, field, value) return meta
def create(self, uri, data, env=None, expectedSize=None, **kwargs): localPath = self._translateUri(uri, **kwargs) self._saveToFile(data,localPath, expectedSize) mime_type, encoding = utils.getMime(localPath) stats = os.stat(localPath) return { '{DAV:}getcontenttype': mime_type, '{DAV:}encoding': encoding, '{DAV:}getcontentlength': stats.st_size, '{DAV:}getlastmodified' : stats.st_mtime, }
def process(self, user, request, resource, path, meta): # the smalles JPEG image is 134 bytes # @see: http://stackoverflow.com/questions/2253404/what-is-the-smallest-valid-jpeg-file-size-in-bytes if not meta: meta = resource.storageProvider.getMeta(path, user=user) meta = meta[path] # don't bother the conversion queue for items that are smaller than minSize bytes size = None try: size = int(meta['{DAV:}getcontentlength']) if size < conf.server.Config['jobclient']['minSize']: return except Exception as ex: logging.getLogger().warn("Incorrect job client configuration: %s" % ex) # Step 0: Deduplicate the data, if the storage provider supports it minDeduplicationSize = 0 try: minDeduplicationSize = conf.server.Config['jobclient']['minDedupSize'] except Exception as ex: logging.getLogger().warn("Incorrect job client configuration: %s" % ex) if size and size > minDeduplicationSize: try: deduplicate = getattr(resource.storageProvider, 'deduplicate') if deduplicate(path, user=user): try: addCache = getattr(resource.storageProvider, 'addCache') # if we use cache decorator -> then refresh the cache for this item addCache(path, user=user) except: pass # This is already processed item -> do not run the conversion for it. return except Exception as ex: logging.getLogger().warn('Deduplicaiton failed: %s, Exception: %s' % (ex, traceback.format_exc())) connection = ConnectionPool().get().getConnection() channel = connection.channel() channel.basic_qos(0,1,False) """ Before we start figure out the exact mime type of the file and some additional properties """ localPath = resource.storageProvider._translateUri(path, user=user) # Step 1: Figure out the mime type mediaInfo = utils.mediaInfo(localPath) mimeType = None if mediaInfo.has_key('General') and mediaInfo['General'].has_key('InternetMediaType'): mimeType = mediaInfo['General']['InternetMediaType'] resource.storageProvider.setMeta(path, {'{DAV:}getcontenttype': mimeType}, user=user) else: [mimeType, encoding] = utils.getMime(localPath) if not mimeType: return # Step 2: For the current user, his/hers conversion group, path and mime-major get the conversions # that can be applied userId = user.get('id') dir = os.path.dirname(path) conversionGroup = user.get('caps:conversion_group') # the convertion group is the default convertions that the user has (major, minor) = mimeType.split('/',1) converter = Conversions() allConversions = { '*': [], major: []} if conversionGroup: groupAllConversions = converter.findRange( "%s-%s-%s" % (conversionGroup, '*', '/'), "%s-%s-%s" % (conversionGroup, '*',dir), ) groupMimeConversions = converter.findRange( "%s-%s-%s" % (conversionGroup, major, '/'), "%s-%s-%s" % (conversionGroup, major, dir), ) allConversions = { '*' : groupAllConversions, major : groupMimeConversions } userAllConversions = converter.findRange( "%s-%s-%s" % (userId, '*', '/'), "%s-%s-%s" % (userId, '*', dir), ) userMimeConversions = converter.findRange( "%s-%s-%s" % (userId, major, '/'), "%s-%s-%s" % (userId, major, dir), ) allConversions['*'] += userAllConversions allConversions[major] += userMimeConversions cipher = coder.SecureLink(**conf.server.Config['secret']) resourceUri = path.replace(request.path,request.uri) tokenServer = 'http://'+TokenVHost.host[0]+':'+TokenVHost.port+'/' tokenUrl = tokenServer + urllib.quote(cipher.encode(['GET','PROPPATCH'], user.getIdentity(), resourceUri)) [viewLocalFolder, viewBaseURL] = resource.storageProvider.getView(path, request, user=user) # create the local base folder where the data will be stored if viewLocalFolder and not os.path.exists(viewLocalFolder): logging.getLogger().debug("Create new view Local Folder: %s" % viewLocalFolder) os.makedirs(viewLocalFolder) jobs = {} for key, convertions in allConversions.items(): for conversion in convertions: actions = {} for (name, value) in conversion.items(): try: if name.find('action:') != -1: action = name[len('action:'):] accepted = conversion.get('accept:'+action) if accepted: accepted = json.loads(accepted) if not minor in accepted: continue actions[action] = json.loads(value) except: logging.getLogger().error('Conversion data is wrong: %s' % traceback.format_exc()) continue for name, options in actions.items(): routingKey = "%s.%s" % (key, name) if not jobs.has_key(routingKey): jobs[routingKey] = { 'url': tokenUrl, 'source': localPath, 'meta' : meta, 'mime' : mimeType, 'targetFolder': viewLocalFolder, 'targetBaseURL': viewBaseURL, 'options': {} } jobs[routingKey]['options'].update(options) for routingKey, job in jobs.items(): logging.getLogger().debug("Sending: RoutingKey:[%s], Message:[%s]" % (routingKey, json.dumps(job))) msg = amqp.Message(json.dumps(job)) msg.properties["delivery_mode"] = 2 channel.basic_publish(msg, exchange='workers', routing_key=routingKey ) channel.close()
def process(self, user, request, resource, path, meta): # the smalles JPEG image is 134 bytes # @see: http://stackoverflow.com/questions/2253404/what-is-the-smallest-valid-jpeg-file-size-in-bytes if not meta: meta = resource.storageProvider.getMeta(path, user=user) meta = meta[path] # don't bother the conversion queue for items that are smaller than minSize bytes size = None try: size = int(meta['{DAV:}getcontentlength']) if size < conf.server.Config['jobclient']['minSize']: return except Exception as ex: logging.getLogger().warn("Incorrect job client configuration: %s" % ex) # Step 0: Deduplicate the data, if the storage provider supports it minDeduplicationSize = 0 try: minDeduplicationSize = conf.server.Config['jobclient'][ 'minDedupSize'] except Exception as ex: logging.getLogger().warn("Incorrect job client configuration: %s" % ex) if size and size > minDeduplicationSize: try: deduplicate = getattr(resource.storageProvider, 'deduplicate') if deduplicate(path, user=user): try: addCache = getattr(resource.storageProvider, 'addCache') # if we use cache decorator -> then refresh the cache for this item addCache(path, user=user) except: pass # This is already processed item -> do not run the conversion for it. return except Exception as ex: logging.getLogger().warn( 'Deduplicaiton failed: %s, Exception: %s' % (ex, traceback.format_exc())) connection = ConnectionPool().get().getConnection() channel = connection.channel() channel.basic_qos(0, 1, False) """ Before we start figure out the exact mime type of the file and some additional properties """ localPath = resource.storageProvider._translateUri(path, user=user) # Step 1: Figure out the mime type mediaInfo = utils.mediaInfo(localPath) mimeType = None if mediaInfo.has_key('General') and mediaInfo['General'].has_key( 'InternetMediaType'): mimeType = mediaInfo['General']['InternetMediaType'] resource.storageProvider.setMeta( path, {'{DAV:}getcontenttype': mimeType}, user=user) else: [mimeType, encoding] = utils.getMime(localPath) if not mimeType: return # Step 2: For the current user, his/hers conversion group, path and mime-major get the conversions # that can be applied userId = user.get('id') dir = os.path.dirname(path) conversionGroup = user.get( 'caps:conversion_group' ) # the convertion group is the default convertions that the user has (major, minor) = mimeType.split('/', 1) converter = Conversions() allConversions = {'*': [], major: []} if conversionGroup: groupAllConversions = converter.findRange( "%s-%s-%s" % (conversionGroup, '*', '/'), "%s-%s-%s" % (conversionGroup, '*', dir), ) groupMimeConversions = converter.findRange( "%s-%s-%s" % (conversionGroup, major, '/'), "%s-%s-%s" % (conversionGroup, major, dir), ) allConversions = { '*': groupAllConversions, major: groupMimeConversions } userAllConversions = converter.findRange( "%s-%s-%s" % (userId, '*', '/'), "%s-%s-%s" % (userId, '*', dir), ) userMimeConversions = converter.findRange( "%s-%s-%s" % (userId, major, '/'), "%s-%s-%s" % (userId, major, dir), ) allConversions['*'] += userAllConversions allConversions[major] += userMimeConversions cipher = coder.SecureLink(**conf.server.Config['secret']) resourceUri = path.replace(request.path, request.uri) tokenServer = 'http://' + TokenVHost.host[ 0] + ':' + TokenVHost.port + '/' tokenUrl = tokenServer + urllib.quote( cipher.encode(['GET', 'PROPPATCH'], user.getIdentity(), resourceUri)) [viewLocalFolder, viewBaseURL] = resource.storageProvider.getView(path, request, user=user) # create the local base folder where the data will be stored if viewLocalFolder and not os.path.exists(viewLocalFolder): logging.getLogger().debug("Create new view Local Folder: %s" % viewLocalFolder) os.makedirs(viewLocalFolder) jobs = {} for key, convertions in allConversions.items(): for conversion in convertions: actions = {} for (name, value) in conversion.items(): try: if name.find('action:') != -1: action = name[len('action:'):] accepted = conversion.get('accept:' + action) if accepted: accepted = json.loads(accepted) if not minor in accepted: continue actions[action] = json.loads(value) except: logging.getLogger().error( 'Conversion data is wrong: %s' % traceback.format_exc()) continue for name, options in actions.items(): routingKey = "%s.%s" % (key, name) if not jobs.has_key(routingKey): jobs[routingKey] = { 'url': tokenUrl, 'source': localPath, 'meta': meta, 'mime': mimeType, 'targetFolder': viewLocalFolder, 'targetBaseURL': viewBaseURL, 'options': {} } jobs[routingKey]['options'].update(options) for routingKey, job in jobs.items(): logging.getLogger().debug( "Sending: RoutingKey:[%s], Message:[%s]" % (routingKey, json.dumps(job))) msg = amqp.Message(json.dumps(job)) msg.properties["delivery_mode"] = 2 channel.basic_publish(msg, exchange='workers', routing_key=routingKey) channel.close()