def load_imsto(section='imsto'): config = Config() engine = config.get('engine', section) print 'loading {} engine: {}'.format(section, engine) if engine == 'mongodb': return StoreEngineGridFs(section) if engine == 's3': return StoreEngineS3(section) if engine == 'weedfs': return StoreEngineWeedFs(section) raise ValueError('bad engine_code')
class StoreBase: engine = None _db = None _fs = None _coll = None def __init__(self, section='imsto'): """engine: mongodb(default), s3""" self.section = section self._config = Config() self.engine = self.get_config('engine') self.fs_prefix = self.get_config('fs_prefix') print 'init section: {self.section}, engine: {self.engine}, fs_prefix: {self.fs_prefix}'.format( self=self) def get_config(self, key): return self._config.get(key, self.section) def browse(self, limit=20, start=0, sort=None, only_items=False): """retrieve files from mongodb for gallery""" #return fs().list() if sort is None or not isinstance(sort, list): sort = [('uploadDate', DESCENDING)] cursor = self.collection.find(limit=limit, skip=start, sort=sort) items = [StoreItem(self, item) for item in cursor] if only_items: return items url_prefix = urljoin(self.get_config('url_prefix'), self.get_config('thumb_path')) return { 'items': items, 'total': cursor.count(), 'url_prefix': url_prefix + '/' } def count(self): return self.collection.count() # def __iter__(self): # self.__cursor = self.collection.find(limit=0,skip=0,sort=[('uploadDate',DESCENDING)]) # return self # def next(self): # if self.__cursor: # return StoreItem(self, self.__cursor.next()) # raise StopIteration def store(self, file=None, content=None, ctype=None, **kwd): """save a file-like item""" if content is None and not hasattr(file, 'read'): raise TypeError('invalid file-like object') data = content if content is not None else file.read() size = len(data) ext = guessImageType(data[:32]) if ext is None: raise ValueError('invalid image file') hashes = [md5(data).hexdigest()] _exists_id = self.exists(hashed=hashes[0]) if _exists_id: id = _exists_id filename = _make_filename(id, ext) print('id {} or hash {} exists!!'.format(id, hashes[0])) #raise DuplicateError('already exists') return [True, id, filename] ids = [_make_id(hashes[0])] if 'id' in kwd and kwd['id'] and kwd['id'] not in ids: ids += [kwd['id']] from image import SimpImage, MIN_QUALITY max_file_size = int(self.get_config('max_file_size')) max_jpeg_quality = int(self.get_config('max_jpeg_quality')) max_width = int(self.get_config('max_width')) max_height = int(self.get_config('max_height')) if size > max_file_size: max_jpeg_quality -= 1 if max_jpeg_quality < MIN_QUALITY: max_jpeg_quality = MIN_QUALITY im = SimpImage(blob=data) meta = im.meta if meta['width'] > max_width or meta['height'] > max_height: if self.get_config('auto_scale') and im.thumbnail( max_width, max_height): if im.format == 'JPEG' and im.quality > max_jpeg_quality: im.quality = max_jpeg_quality data = im.get_blob() size = len(data) print im.meta print 'new scaled size {}'.format(size) hashes += [md5(data).hexdigest()] else: raise ValueError( 'file: {} dimension {}x{} is too big, max is {}x{}'.format( kwd['name'] if 'name' in kwd else '', meta['width'], meta['height'], max_width, max_height)) if im.format == 'JPEG': if im.quality > max_jpeg_quality: print 'quality {} is too high, hash {}'.format( im.quality, hashes[0]) from tempfile import NamedTemporaryFile _tmp = NamedTemporaryFile('w+b', dir=self.get_config('temp_root'), delete=False) _tmp.file.close() save_file(_tmp.name, blob=data) if jpegoptim(_tmp.name): fp = open(_tmp.name) data = fp.read() size = len(data) # print 'new optimized size {}'.format(size) fp.close() _tmp.unlink(_tmp.name) del im im = SimpImage(blob=data) meta = im.meta hashes += [md5(data).hexdigest()] else: raise EnvironmentError( 'jpeg qualty is too high, or need jpegoptim') elif im.format == 'PNG' and self.get_config('force_jpeg'): im.format = 'JPEG' im.quality = max_jpeg_quality data = im.get_blob() size = len(data) hashes += [md5(data).hexdigest()] ext = 'jpg' meta = im.meta del im if (size > max_file_size): raise ValueError('file: {} size {} is too big, max is {}'.format( kwd['name'] if 'name' in kwd else '', size, max_file_size)) hashed = hashes[len(hashes) - 1] #md5(data).hexdigest() # print ('md5 hash: {}'.format(hashed)) # TODO: add for support (md5 + size) id id = _make_id(hashed) # print ('new filename: %r' % filename) # TODO: fix for support s3 front browse _exists_id = self.exists(id) or self.exists(hashed=hashed) if _exists_id: id = _exists_id filename = _make_filename(id, ext) print('id {} or hash {} exists!!'.format(id, hashed)) #raise DuplicateError('already exists') return [True, id, filename] filename = _make_filename(id, ext) # print ('id: {}'.format(id)) # if ctype is None or ctype == '': from _util import guess_mimetype ctype = guess_mimetype(filename) # save to mongodb spec = { '_id': id, 'filename': filename, 'hash': hashes, 'mime': ctype, 'size': size, 'meta': meta, 'ids': ids } if 'name' in kwd and isinstance(kwd['name'], (str, unicode)): spec['name'] = kwd['name'] for k in ['created', 'app_id']: if k in kwd and kwd[k]: spec[k] = kwd[k] if self._store_exists(id, filename=filename): self._save_meta(id, spec) return [True, id, filename] rr = self._put(data, **spec) if rr: return [True, rr, filename] def get_meta(self, id=None, filename=None, ids=None): spec = None if id: spec = id elif filename: spec = {'filename': filename} elif ids and isinstance(ids, type([])): spec = {'ids': {'$in': ids}} if spec: print 'spec %s' % spec item = self.collection.find_one(spec) if item: return StoreItem(self, item) def _save_meta(self, id, spec): '''mongo special meta data''' #if not hasattr(spec, '_id'): # spec['_id'] = id if 'created' not in spec: spec['created'] = datetime.datetime.utcnow() if 'filename' not in spec: print spec raise ValueError('need filename') return self.collection.update({'_id': id}, spec, upsert=True) def delete(self, id): raise NotImplemented() def _get(self, id): raise NotImplemented() def _put(self, data, **spec): raise NotImplemented() def _store_exists(self, id=None, *args, **kwargs): raise NotImplemented() def exists(self, id=None, hashed=None, filename=None, *args, **kwargs): """check special hash value TODO: more args""" #print args #print kwargs if id and self.collection.find_one({"_id": id}): return id if hashed: doc = self.collection.find_one({'md5': hashed}) if doc: return doc['_id'] doc = self.collection.find_one({'hash': {'$in': [hashed]}}) if doc: return doc['_id'] if filename: doc = self.collection.find_one(filename=filename) if doc: return doc['_id'] if self._store_exists(id, hashed=hashed, filename=filename, *args, **kwargs): return id @property def db(self): if self._db is None: self._db = get_mongo_db(self.get_config('servers'), self.get_config('db_name'), self.get_config('replica_set')) return self._db @property def collection(self): if self._coll is None: cn = '{0}.files'.format(self.fs_prefix) self._coll = self.db[cn] return self._coll def close(self): """ close db connection""" if self.db is not None: self.db.connection.disconnect() def load(self, path): """ load from url path """ #print 'path: %s (%s)' % (path, type(path)) image_url_regex = r'(?P<size>[scwh]\d{2,4}(?P<x>x\d{2,4})?|orig)(?P<mop>[a-z])?/(?P<t1>[a-z0-9]{2})/(?P<t2>[a-z0-9]{2})/(?P<t3>[a-z0-9]{19,36})\.(?P<ext>gif|jpg|jpeg|png)$' match = re.search(image_url_regex, path) #print(image_url_regex, path, match) if match is None: raise UrlError('invalid path') ids = match.groupdict() #print(ids) id = '{t1}{t2}{t3}'.format(**ids) THUMB_ROOT = self.get_config('thumb_root').rstrip('/') SUPPORTED_SIZE = self.get_config('support_size').split(',') org_path = '{t1}/{t2}/{t3}.{ext}'.format(**ids) org_file = '{0}/orig/{1}'.format(THUMB_ROOT, org_path) if not os.path.exists(org_file): # check old id for redirect doc = self.get_meta(ids=[id]) if doc and doc['id'] != id and 'filename' in doc: print 'found %s' % doc['filename'] thumb_path = self.get_config('thumb_path') new_path = '{}/{}/{}'.format(thumb_path, ids['size'], doc['filename']) raise HttpFound('found', path=new_path) print('fetching file: {}'.format(org_path)) file = self.fetch(id, path=org_path) if file is None: print('fetch failed') raise UrlError('id {} not found'.format(id)) save_file(org_file, file) if not os.path.exists(org_file): raise UrlError('file not found') # start thumbnail image if ids['size'] == 'orig': dst_path = 'orig/{}'.format(org_path) dst_file = org_file else: dst_path = '{0}/{1}'.format(ids['size'], org_path) dst_file = '{0}/{1}'.format(THUMB_ROOT, dst_path) mode = ids['size'][0] dimension = ids['size'][1:] if dimension not in SUPPORTED_SIZE: #print('unsupported size: {} {}'.format(mode, dimension)) raise UrlError('unsupported size') if ids['x'] is None: size = int(dimension) width, height = size, size else: width, height = map(int, dimension.split('x')) if not os.path.exists(dst_file): print('start thumbnail image {} {} => {}x{}'.format( mode, dimension, width, height)) thumb_image(org_file, width, height, dst_file, mode) if ids['mop'] == 'w' and width < 100: raise UrlError('bad size') if ids['mop'] is not None: if ids['mop'] == 'w': # watermark modifier org_file = '{}/{}/{}'.format(THUMB_ROOT, ids['size'], org_path) dst_file = '{}/{}{}/{}'.format(THUMB_ROOT, ids['size'], ids['mop'], org_path) if watermark_image(org_file, dst_file): dst_path = '{}{}/{}'.format(ids['size'], ids['mop'], org_path) else: raise UrlError('bad modifier') #print('dst_path: {}'.format(dst_path)) #print('dst_file: {}'.format(dst_file)) return (dst_file, dst_path) def fetch(self, id, path): key = path if self.engine == 's3' else id return self._get(key) # try: # return self._get(key) # except Exception, e: # print('prepare: {} not found'.format(key)) # print e # raise e def url(self, path, size='orig'): url_prefix = self.get_config('url_prefix') thumb_path = self.get_config('thumb_path') return '{}/{}/{}/{}'.format(url_prefix.rstrip('/'), thumb_path.strip('/'), size, path)
class StoreBase: engine = None _db = None _fs = None _coll = None def __init__(self, section='imsto'): """engine: mongodb(default), s3""" self.section = section self._config = Config() self.engine = self.get_config('engine') self.fs_prefix = self.get_config('fs_prefix') print 'init section: {self.section}, engine: {self.engine}, fs_prefix: {self.fs_prefix}'.format(self=self) def get_config(self, key): return self._config.get(key, self.section) def browse(self, limit=20, start=0, sort=None, only_items = False): """retrieve files from mongodb for gallery""" #return fs().list() if sort is None or not isinstance(sort, list): sort = [('uploadDate',DESCENDING)] cursor = self.collection.find(limit=limit,skip=start,sort=sort) items = [StoreItem(self, item) for item in cursor] if only_items: return items url_prefix = urljoin(self.get_config('url_prefix'), self.get_config('thumb_path')) return {'items':items,'total':cursor.count(),'url_prefix': url_prefix + '/'} def count(self): return self.collection.count(); # def __iter__(self): # self.__cursor = self.collection.find(limit=0,skip=0,sort=[('uploadDate',DESCENDING)]) # return self # def next(self): # if self.__cursor: # return StoreItem(self, self.__cursor.next()) # raise StopIteration def store(self, file=None, content=None, ctype=None, **kwd): """save a file-like item""" if content is None and not hasattr(file, 'read'): raise TypeError('invalid file-like object') data = content if content is not None else file.read() size = len(data) ext = guessImageType(data[:32]) if ext is None: raise ValueError('invalid image file') hashes = [md5(data).hexdigest()] _exists_id = self.exists(hashed=hashes[0]) if _exists_id: id = _exists_id filename = _make_filename(id, ext) print ('id {} or hash {} exists!!'.format(id, hashes[0])) #raise DuplicateError('already exists') return [True, id, filename] ids = [_make_id(hashes[0])] if 'id' in kwd and kwd['id'] and kwd['id'] not in ids: ids += [kwd['id']] from image import SimpImage, MIN_QUALITY max_file_size = int(self.get_config('max_file_size')) max_jpeg_quality = int(self.get_config('max_jpeg_quality')) max_width = int(self.get_config('max_width')) max_height = int(self.get_config('max_height')) if size > max_file_size: max_jpeg_quality -= 1 if max_jpeg_quality < MIN_QUALITY: max_jpeg_quality = MIN_QUALITY im = SimpImage(blob=data) meta = im.meta if meta['width'] > max_width or meta['height'] > max_height: if self.get_config('auto_scale') and im.thumbnail(max_width, max_height): if im.format == 'JPEG' and im.quality > max_jpeg_quality: im.quality = max_jpeg_quality data = im.get_blob() size = len(data) print im.meta print 'new scaled size {}'.format(size) hashes += [md5(data).hexdigest()] else: raise ValueError('file: {} dimension {}x{} is too big, max is {}x{}'.format(kwd['name'] if 'name' in kwd else '', meta['width'], meta['height'], max_width, max_height)) if im.format == 'JPEG': if im.quality > max_jpeg_quality: print 'quality {} is too high, hash {}'.format(im.quality, hashes[0]) from tempfile import NamedTemporaryFile _tmp = NamedTemporaryFile('w+b',dir=self.get_config('temp_root'),delete=False) _tmp.file.close() save_file(_tmp.name, blob=data) if jpegoptim(_tmp.name): fp = open(_tmp.name) data = fp.read() size = len(data) # print 'new optimized size {}'.format(size) fp.close() _tmp.unlink(_tmp.name) del im im = SimpImage(blob=data) meta = im.meta hashes += [md5(data).hexdigest()] else: raise EnvironmentError('jpeg qualty is too high, or need jpegoptim') elif im.format == 'PNG' and self.get_config('force_jpeg'): im.format = 'JPEG' im.quality = max_jpeg_quality data = im.get_blob() size = len(data) hashes += [md5(data).hexdigest()] ext = 'jpg' meta = im.meta del im if (size > max_file_size): raise ValueError('file: {} size {} is too big, max is {}'.format(kwd['name'] if 'name' in kwd else '', size, max_file_size)) hashed = hashes[len(hashes)-1] #md5(data).hexdigest() # print ('md5 hash: {}'.format(hashed)) # TODO: add for support (md5 + size) id id = _make_id(hashed) # print ('new filename: %r' % filename) # TODO: fix for support s3 front browse _exists_id = self.exists(id) or self.exists(hashed=hashed) if _exists_id: id = _exists_id filename = _make_filename(id, ext) print ('id {} or hash {} exists!!'.format(id, hashed)) #raise DuplicateError('already exists') return [True, id, filename] filename = _make_filename(id, ext) # print ('id: {}'.format(id)) # if ctype is None or ctype == '': from _util import guess_mimetype ctype = guess_mimetype(filename) # save to mongodb spec = {'_id': id,'filename': filename, 'hash': hashes, 'mime': ctype, 'size': size, 'meta': meta, 'ids': ids} if 'name' in kwd and isinstance(kwd['name'], (str, unicode)): spec['name'] = kwd['name'] for k in ['created', 'app_id']: if k in kwd and kwd[k]: spec[k] = kwd[k] if self._store_exists(id, filename=filename): self._save_meta(id, spec) return [True, id, filename] rr = self._put(data, **spec) if rr: return [True, rr, filename] def get_meta(self, id=None, filename=None, ids=None): spec = None if id: spec = id elif filename: spec = {'filename': filename} elif ids and isinstance(ids, type([])): spec = {'ids': {'$in': ids}} if spec: print 'spec %s' % spec item = self.collection.find_one(spec) if item: return StoreItem(self, item) def _save_meta(self, id, spec): '''mongo special meta data''' #if not hasattr(spec, '_id'): # spec['_id'] = id if 'created' not in spec: spec['created'] = datetime.datetime.utcnow() if 'filename' not in spec: print spec raise ValueError('need filename') return self.collection.update({'_id': id}, spec, upsert=True) def delete(self, id): raise NotImplemented() def _get(self, id): raise NotImplemented() def _put(self, data, **spec): raise NotImplemented() def _store_exists(self, id=None, *args, **kwargs): raise NotImplemented() def exists(self, id=None, hashed=None, filename=None, *args, **kwargs): """check special hash value TODO: more args""" #print args #print kwargs if id and self.collection.find_one({"_id": id}): return id if hashed: doc = self.collection.find_one({'md5': hashed}) if doc: return doc['_id'] doc = self.collection.find_one({'hash': {'$in': [hashed]}}) if doc: return doc['_id'] if filename: doc = self.collection.find_one(filename=filename) if doc: return doc['_id'] if self._store_exists(id, hashed=hashed, filename=filename, *args, **kwargs): return id @property def db(self): if self._db is None: self._db = get_mongo_db(self.get_config('servers'), self.get_config('db_name'), self.get_config('replica_set')) return self._db @property def collection(self): if self._coll is None: cn = '{0}.files'.format(self.fs_prefix) self._coll = self.db[cn] return self._coll def close(self): """ close db connection""" if self.db is not None: self.db.connection.disconnect() def load(self, path): """ load from url path """ #print 'path: %s (%s)' % (path, type(path)) image_url_regex = r'(?P<size>[scwh]\d{2,4}(?P<x>x\d{2,4})?|orig)(?P<mop>[a-z])?/(?P<t1>[a-z0-9]{2})/(?P<t2>[a-z0-9]{2})/(?P<t3>[a-z0-9]{19,36})\.(?P<ext>gif|jpg|jpeg|png)$' match = re.search(image_url_regex, path) #print(image_url_regex, path, match) if match is None: raise UrlError('invalid path') ids = match.groupdict() #print(ids) id = '{t1}{t2}{t3}'.format(**ids) THUMB_ROOT = self.get_config('thumb_root').rstrip('/') SUPPORTED_SIZE = self.get_config('support_size').split(',') org_path = '{t1}/{t2}/{t3}.{ext}'.format(**ids) org_file = '{0}/orig/{1}'.format(THUMB_ROOT, org_path) if not os.path.exists(org_file): # check old id for redirect doc = self.get_meta(ids=[id]) if doc and doc['id'] != id and 'filename' in doc: print 'found %s' % doc['filename'] thumb_path = self.get_config('thumb_path') new_path = '{}/{}/{}'.format(thumb_path, ids['size'], doc['filename']) raise HttpFound('found', path=new_path) print('fetching file: {}'.format(org_path)) file = self.fetch(id, path=org_path) if file is None: print('fetch failed') raise UrlError('id {} not found'.format(id)) save_file(org_file, file) if not os.path.exists(org_file): raise UrlError('file not found') # start thumbnail image if ids['size'] == 'orig': dst_path = 'orig/{}'.format(org_path) dst_file = org_file else: dst_path = '{0}/{1}'.format(ids['size'], org_path) dst_file = '{0}/{1}'.format(THUMB_ROOT, dst_path) mode = ids['size'][0] dimension = ids['size'][1:] if dimension not in SUPPORTED_SIZE: #print('unsupported size: {} {}'.format(mode, dimension)) raise UrlError('unsupported size') if ids['x'] is None: size = int(dimension) width, height = size, size else: width, height = map(int, dimension.split('x')) if not os.path.exists(dst_file): print('start thumbnail image {} {} => {}x{}'.format(mode, dimension, width, height)) thumb_image(org_file, width, height, dst_file, mode) if ids['mop'] == 'w' and width < 100: raise UrlError('bad size') if ids['mop'] is not None: if ids['mop'] == 'w': # watermark modifier org_file = '{}/{}/{}'.format(THUMB_ROOT, ids['size'], org_path) dst_file = '{}/{}{}/{}'.format(THUMB_ROOT, ids['size'], ids['mop'], org_path) if watermark_image(org_file, dst_file): dst_path = '{}{}/{}'.format(ids['size'], ids['mop'], org_path) else: raise UrlError('bad modifier') #print('dst_path: {}'.format(dst_path)) #print('dst_file: {}'.format(dst_file)) return (dst_file, dst_path) def fetch(self, id, path): key = path if self.engine == 's3' else id return self._get(key) # try: # return self._get(key) # except Exception, e: # print('prepare: {} not found'.format(key)) # print e # raise e def url(self, path, size='orig'): url_prefix = self.get_config('url_prefix') thumb_path = self.get_config('thumb_path') return '{}/{}/{}/{}'.format(url_prefix.rstrip('/'), thumb_path.strip('/'), size, path)