def load_imsto(section='imsto'):
config = Config()
engine = config.get('engine', section)
print 'loading {} engine: {}'.format(section, engine)
if engine == 'mongodb':
return StoreEngineGridFs(section)
if engine == 's3':
return StoreEngineS3(section)
if engine == 'weedfs':
return StoreEngineWeedFs(section)
raise ValueError('bad engine_code')
def load_imsto(section='imsto'):
config = Config()
engine = config.get('engine', section)
print 'loading {} engine: {}'.format(section, engine)
if engine == 'mongodb':
return StoreEngineGridFs(section)
if engine == 's3':
return StoreEngineS3(section)
if engine == 'weedfs':
return StoreEngineWeedFs(section)
raise ValueError('bad engine_code')
class StoreBase:
engine = None
_db = None
_fs = None
_coll = None
def __init__(self, section='imsto'):
"""engine: mongodb(default), s3"""
self.section = section
self._config = Config()
self.engine = self.get_config('engine')
self.fs_prefix = self.get_config('fs_prefix')
print 'init section: {self.section}, engine: {self.engine}, fs_prefix: {self.fs_prefix}'.format(
self=self)
def get_config(self, key):
return self._config.get(key, self.section)
def browse(self, limit=20, start=0, sort=None, only_items=False):
"""retrieve files from mongodb for gallery"""
#return fs().list()
if sort is None or not isinstance(sort, list):
sort = [('uploadDate', DESCENDING)]
cursor = self.collection.find(limit=limit, skip=start, sort=sort)
items = [StoreItem(self, item) for item in cursor]
if only_items:
return items
url_prefix = urljoin(self.get_config('url_prefix'),
self.get_config('thumb_path'))
return {
'items': items,
'total': cursor.count(),
'url_prefix': url_prefix + '/'
}
def count(self):
return self.collection.count()
# def __iter__(self):
# self.__cursor = self.collection.find(limit=0,skip=0,sort=[('uploadDate',DESCENDING)])
# return self
# def next(self):
# if self.__cursor:
# return StoreItem(self, self.__cursor.next())
# raise StopIteration
def store(self, file=None, content=None, ctype=None, **kwd):
"""save a file-like item"""
if content is None and not hasattr(file, 'read'):
raise TypeError('invalid file-like object')
data = content if content is not None else file.read()
size = len(data)
ext = guessImageType(data[:32])
if ext is None:
raise ValueError('invalid image file')
hashes = [md5(data).hexdigest()]
_exists_id = self.exists(hashed=hashes[0])
if _exists_id:
id = _exists_id
filename = _make_filename(id, ext)
print('id {} or hash {} exists!!'.format(id, hashes[0]))
#raise DuplicateError('already exists')
return [True, id, filename]
ids = [_make_id(hashes[0])]
if 'id' in kwd and kwd['id'] and kwd['id'] not in ids:
ids += [kwd['id']]
from image import SimpImage, MIN_QUALITY
max_file_size = int(self.get_config('max_file_size'))
max_jpeg_quality = int(self.get_config('max_jpeg_quality'))
max_width = int(self.get_config('max_width'))
max_height = int(self.get_config('max_height'))
if size > max_file_size: max_jpeg_quality -= 1
if max_jpeg_quality < MIN_QUALITY: max_jpeg_quality = MIN_QUALITY
im = SimpImage(blob=data)
meta = im.meta
if meta['width'] > max_width or meta['height'] > max_height:
if self.get_config('auto_scale') and im.thumbnail(
max_width, max_height):
if im.format == 'JPEG' and im.quality > max_jpeg_quality:
im.quality = max_jpeg_quality
data = im.get_blob()
size = len(data)
print im.meta
print 'new scaled size {}'.format(size)
hashes += [md5(data).hexdigest()]
else:
raise ValueError(
'file: {} dimension {}x{} is too big, max is {}x{}'.format(
kwd['name'] if 'name' in kwd else '', meta['width'],
meta['height'], max_width, max_height))
if im.format == 'JPEG':
if im.quality > max_jpeg_quality:
print 'quality {} is too high, hash {}'.format(
im.quality, hashes[0])
from tempfile import NamedTemporaryFile
_tmp = NamedTemporaryFile('w+b',
dir=self.get_config('temp_root'),
delete=False)
_tmp.file.close()
save_file(_tmp.name, blob=data)
if jpegoptim(_tmp.name):
fp = open(_tmp.name)
data = fp.read()
size = len(data)
# print 'new optimized size {}'.format(size)
fp.close()
_tmp.unlink(_tmp.name)
del im
im = SimpImage(blob=data)
meta = im.meta
hashes += [md5(data).hexdigest()]
else:
raise EnvironmentError(
'jpeg qualty is too high, or need jpegoptim')
elif im.format == 'PNG' and self.get_config('force_jpeg'):
im.format = 'JPEG'
im.quality = max_jpeg_quality
data = im.get_blob()
size = len(data)
hashes += [md5(data).hexdigest()]
ext = 'jpg'
meta = im.meta
del im
if (size > max_file_size):
raise ValueError('file: {} size {} is too big, max is {}'.format(
kwd['name'] if 'name' in kwd else '', size, max_file_size))
hashed = hashes[len(hashes) - 1] #md5(data).hexdigest()
# print ('md5 hash: {}'.format(hashed))
# TODO: add for support (md5 + size) id
id = _make_id(hashed)
# print ('new filename: %r' % filename)
# TODO: fix for support s3 front browse
_exists_id = self.exists(id) or self.exists(hashed=hashed)
if _exists_id:
id = _exists_id
filename = _make_filename(id, ext)
print('id {} or hash {} exists!!'.format(id, hashed))
#raise DuplicateError('already exists')
return [True, id, filename]
filename = _make_filename(id, ext)
# print ('id: {}'.format(id))
# if ctype is None or ctype == '':
from _util import guess_mimetype
ctype = guess_mimetype(filename)
# save to mongodb
spec = {
'_id': id,
'filename': filename,
'hash': hashes,
'mime': ctype,
'size': size,
'meta': meta,
'ids': ids
}
if 'name' in kwd and isinstance(kwd['name'], (str, unicode)):
spec['name'] = kwd['name']
for k in ['created', 'app_id']:
if k in kwd and kwd[k]:
spec[k] = kwd[k]
if self._store_exists(id, filename=filename):
self._save_meta(id, spec)
return [True, id, filename]
rr = self._put(data, **spec)
if rr:
return [True, rr, filename]
def get_meta(self, id=None, filename=None, ids=None):
spec = None
if id:
spec = id
elif filename:
spec = {'filename': filename}
elif ids and isinstance(ids, type([])):
spec = {'ids': {'$in': ids}}
if spec:
print 'spec %s' % spec
item = self.collection.find_one(spec)
if item:
return StoreItem(self, item)
def _save_meta(self, id, spec):
'''mongo special meta data'''
#if not hasattr(spec, '_id'):
# spec['_id'] = id
if 'created' not in spec:
spec['created'] = datetime.datetime.utcnow()
if 'filename' not in spec:
print spec
raise ValueError('need filename')
return self.collection.update({'_id': id}, spec, upsert=True)
def delete(self, id):
raise NotImplemented()
def _get(self, id):
raise NotImplemented()
def _put(self, data, **spec):
raise NotImplemented()
def _store_exists(self, id=None, *args, **kwargs):
raise NotImplemented()
def exists(self, id=None, hashed=None, filename=None, *args, **kwargs):
"""check special hash value TODO: more args"""
#print args
#print kwargs
if id and self.collection.find_one({"_id": id}):
return id
if hashed:
doc = self.collection.find_one({'md5': hashed})
if doc:
return doc['_id']
doc = self.collection.find_one({'hash': {'$in': [hashed]}})
if doc:
return doc['_id']
if filename:
doc = self.collection.find_one(filename=filename)
if doc:
return doc['_id']
if self._store_exists(id,
hashed=hashed,
filename=filename,
*args,
**kwargs):
return id
@property
def db(self):
if self._db is None:
self._db = get_mongo_db(self.get_config('servers'),
self.get_config('db_name'),
self.get_config('replica_set'))
return self._db
@property
def collection(self):
if self._coll is None:
cn = '{0}.files'.format(self.fs_prefix)
self._coll = self.db[cn]
return self._coll
def close(self):
""" close db connection"""
if self.db is not None:
self.db.connection.disconnect()
def load(self, path):
""" load from url path """
#print 'path: %s (%s)' % (path, type(path))
image_url_regex = r'(?P<size>[scwh]\d{2,4}(?P<x>x\d{2,4})?|orig)(?P<mop>[a-z])?/(?P<t1>[a-z0-9]{2})/(?P<t2>[a-z0-9]{2})/(?P<t3>[a-z0-9]{19,36})\.(?P<ext>gif|jpg|jpeg|png)$'
match = re.search(image_url_regex, path)
#print(image_url_regex, path, match)
if match is None:
raise UrlError('invalid path')
ids = match.groupdict()
#print(ids)
id = '{t1}{t2}{t3}'.format(**ids)
THUMB_ROOT = self.get_config('thumb_root').rstrip('/')
SUPPORTED_SIZE = self.get_config('support_size').split(',')
org_path = '{t1}/{t2}/{t3}.{ext}'.format(**ids)
org_file = '{0}/orig/{1}'.format(THUMB_ROOT, org_path)
if not os.path.exists(org_file):
# check old id for redirect
doc = self.get_meta(ids=[id])
if doc and doc['id'] != id and 'filename' in doc:
print 'found %s' % doc['filename']
thumb_path = self.get_config('thumb_path')
new_path = '{}/{}/{}'.format(thumb_path, ids['size'],
doc['filename'])
raise HttpFound('found', path=new_path)
print('fetching file: {}'.format(org_path))
file = self.fetch(id, path=org_path)
if file is None:
print('fetch failed')
raise UrlError('id {} not found'.format(id))
save_file(org_file, file)
if not os.path.exists(org_file):
raise UrlError('file not found')
# start thumbnail image
if ids['size'] == 'orig':
dst_path = 'orig/{}'.format(org_path)
dst_file = org_file
else:
dst_path = '{0}/{1}'.format(ids['size'], org_path)
dst_file = '{0}/{1}'.format(THUMB_ROOT, dst_path)
mode = ids['size'][0]
dimension = ids['size'][1:]
if dimension not in SUPPORTED_SIZE:
#print('unsupported size: {} {}'.format(mode, dimension))
raise UrlError('unsupported size')
if ids['x'] is None:
size = int(dimension)
width, height = size, size
else:
width, height = map(int, dimension.split('x'))
if not os.path.exists(dst_file):
print('start thumbnail image {} {} => {}x{}'.format(
mode, dimension, width, height))
thumb_image(org_file, width, height, dst_file, mode)
if ids['mop'] == 'w' and width < 100:
raise UrlError('bad size')
if ids['mop'] is not None:
if ids['mop'] == 'w': # watermark modifier
org_file = '{}/{}/{}'.format(THUMB_ROOT, ids['size'], org_path)
dst_file = '{}/{}{}/{}'.format(THUMB_ROOT, ids['size'],
ids['mop'], org_path)
if watermark_image(org_file, dst_file):
dst_path = '{}{}/{}'.format(ids['size'], ids['mop'],
org_path)
else:
raise UrlError('bad modifier')
#print('dst_path: {}'.format(dst_path))
#print('dst_file: {}'.format(dst_file))
return (dst_file, dst_path)
def fetch(self, id, path):
key = path if self.engine == 's3' else id
return self._get(key)
# try:
# return self._get(key)
# except Exception, e:
# print('prepare: {} not found'.format(key))
# print e
# raise e
def url(self, path, size='orig'):
url_prefix = self.get_config('url_prefix')
thumb_path = self.get_config('thumb_path')
return '{}/{}/{}/{}'.format(url_prefix.rstrip('/'),
thumb_path.strip('/'), size, path)
class StoreBase:
engine = None
_db = None
_fs = None
_coll = None
def __init__(self, section='imsto'):
"""engine: mongodb(default), s3"""
self.section = section
self._config = Config()
self.engine = self.get_config('engine')
self.fs_prefix = self.get_config('fs_prefix')
print 'init section: {self.section}, engine: {self.engine}, fs_prefix: {self.fs_prefix}'.format(self=self)
def get_config(self, key):
return self._config.get(key, self.section)
def browse(self, limit=20, start=0, sort=None, only_items = False):
"""retrieve files from mongodb for gallery"""
#return fs().list()
if sort is None or not isinstance(sort, list):
sort = [('uploadDate',DESCENDING)]
cursor = self.collection.find(limit=limit,skip=start,sort=sort)
items = [StoreItem(self, item) for item in cursor]
if only_items:
return items
url_prefix = urljoin(self.get_config('url_prefix'), self.get_config('thumb_path'))
return {'items':items,'total':cursor.count(),'url_prefix': url_prefix + '/'}
def count(self):
return self.collection.count();
# def __iter__(self):
# self.__cursor = self.collection.find(limit=0,skip=0,sort=[('uploadDate',DESCENDING)])
# return self
# def next(self):
# if self.__cursor:
# return StoreItem(self, self.__cursor.next())
# raise StopIteration
def store(self, file=None, content=None, ctype=None, **kwd):
"""save a file-like item"""
if content is None and not hasattr(file, 'read'):
raise TypeError('invalid file-like object')
data = content if content is not None else file.read()
size = len(data)
ext = guessImageType(data[:32])
if ext is None:
raise ValueError('invalid image file')
hashes = [md5(data).hexdigest()]
_exists_id = self.exists(hashed=hashes[0])
if _exists_id:
id = _exists_id
filename = _make_filename(id, ext)
print ('id {} or hash {} exists!!'.format(id, hashes[0]))
#raise DuplicateError('already exists')
return [True, id, filename]
ids = [_make_id(hashes[0])]
if 'id' in kwd and kwd['id'] and kwd['id'] not in ids:
ids += [kwd['id']]
from image import SimpImage, MIN_QUALITY
max_file_size = int(self.get_config('max_file_size'))
max_jpeg_quality = int(self.get_config('max_jpeg_quality'))
max_width = int(self.get_config('max_width'))
max_height = int(self.get_config('max_height'))
if size > max_file_size: max_jpeg_quality -= 1
if max_jpeg_quality < MIN_QUALITY: max_jpeg_quality = MIN_QUALITY
im = SimpImage(blob=data)
meta = im.meta
if meta['width'] > max_width or meta['height'] > max_height:
if self.get_config('auto_scale') and im.thumbnail(max_width, max_height):
if im.format == 'JPEG' and im.quality > max_jpeg_quality:
im.quality = max_jpeg_quality
data = im.get_blob()
size = len(data)
print im.meta
print 'new scaled size {}'.format(size)
hashes += [md5(data).hexdigest()]
else:
raise ValueError('file: {} dimension {}x{} is too big, max is {}x{}'.format(kwd['name'] if 'name' in kwd else '', meta['width'], meta['height'], max_width, max_height))
if im.format == 'JPEG':
if im.quality > max_jpeg_quality:
print 'quality {} is too high, hash {}'.format(im.quality, hashes[0])
from tempfile import NamedTemporaryFile
_tmp = NamedTemporaryFile('w+b',dir=self.get_config('temp_root'),delete=False)
_tmp.file.close()
save_file(_tmp.name, blob=data)
if jpegoptim(_tmp.name):
fp = open(_tmp.name)
data = fp.read()
size = len(data)
# print 'new optimized size {}'.format(size)
fp.close()
_tmp.unlink(_tmp.name)
del im
im = SimpImage(blob=data)
meta = im.meta
hashes += [md5(data).hexdigest()]
else:
raise EnvironmentError('jpeg qualty is too high, or need jpegoptim')
elif im.format == 'PNG' and self.get_config('force_jpeg'):
im.format = 'JPEG'
im.quality = max_jpeg_quality
data = im.get_blob()
size = len(data)
hashes += [md5(data).hexdigest()]
ext = 'jpg'
meta = im.meta
del im
if (size > max_file_size):
raise ValueError('file: {} size {} is too big, max is {}'.format(kwd['name'] if 'name' in kwd else '', size, max_file_size))
hashed = hashes[len(hashes)-1] #md5(data).hexdigest()
# print ('md5 hash: {}'.format(hashed))
# TODO: add for support (md5 + size) id
id = _make_id(hashed)
# print ('new filename: %r' % filename)
# TODO: fix for support s3 front browse
_exists_id = self.exists(id) or self.exists(hashed=hashed)
if _exists_id:
id = _exists_id
filename = _make_filename(id, ext)
print ('id {} or hash {} exists!!'.format(id, hashed))
#raise DuplicateError('already exists')
return [True, id, filename]
filename = _make_filename(id, ext)
# print ('id: {}'.format(id))
# if ctype is None or ctype == '':
from _util import guess_mimetype
ctype = guess_mimetype(filename)
# save to mongodb
spec = {'_id': id,'filename': filename, 'hash': hashes, 'mime': ctype, 'size': size, 'meta': meta, 'ids': ids}
if 'name' in kwd and isinstance(kwd['name'], (str, unicode)):
spec['name'] = kwd['name']
for k in ['created', 'app_id']:
if k in kwd and kwd[k]:
spec[k] = kwd[k]
if self._store_exists(id, filename=filename):
self._save_meta(id, spec)
return [True, id, filename]
rr = self._put(data, **spec)
if rr:
return [True, rr, filename]
def get_meta(self, id=None, filename=None, ids=None):
spec = None
if id:
spec = id
elif filename:
spec = {'filename': filename}
elif ids and isinstance(ids, type([])):
spec = {'ids': {'$in': ids}}
if spec:
print 'spec %s' % spec
item = self.collection.find_one(spec)
if item:
return StoreItem(self, item)
def _save_meta(self, id, spec):
'''mongo special meta data'''
#if not hasattr(spec, '_id'):
# spec['_id'] = id
if 'created' not in spec:
spec['created'] = datetime.datetime.utcnow()
if 'filename' not in spec:
print spec
raise ValueError('need filename')
return self.collection.update({'_id': id}, spec, upsert=True)
def delete(self, id):
raise NotImplemented()
def _get(self, id):
raise NotImplemented()
def _put(self, data, **spec):
raise NotImplemented()
def _store_exists(self, id=None, *args, **kwargs):
raise NotImplemented()
def exists(self, id=None, hashed=None, filename=None, *args, **kwargs):
"""check special hash value TODO: more args"""
#print args
#print kwargs
if id and self.collection.find_one({"_id": id}):
return id
if hashed:
doc = self.collection.find_one({'md5': hashed})
if doc:
return doc['_id']
doc = self.collection.find_one({'hash': {'$in': [hashed]}})
if doc:
return doc['_id']
if filename:
doc = self.collection.find_one(filename=filename)
if doc:
return doc['_id']
if self._store_exists(id, hashed=hashed, filename=filename, *args, **kwargs):
return id
@property
def db(self):
if self._db is None:
self._db = get_mongo_db(self.get_config('servers'), self.get_config('db_name'), self.get_config('replica_set'))
return self._db
@property
def collection(self):
if self._coll is None:
cn = '{0}.files'.format(self.fs_prefix)
self._coll = self.db[cn]
return self._coll
def close(self):
""" close db connection"""
if self.db is not None:
self.db.connection.disconnect()
def load(self, path):
""" load from url path """
#print 'path: %s (%s)' % (path, type(path))
image_url_regex = r'(?P<size>[scwh]\d{2,4}(?P<x>x\d{2,4})?|orig)(?P<mop>[a-z])?/(?P<t1>[a-z0-9]{2})/(?P<t2>[a-z0-9]{2})/(?P<t3>[a-z0-9]{19,36})\.(?P<ext>gif|jpg|jpeg|png)$'
match = re.search(image_url_regex, path)
#print(image_url_regex, path, match)
if match is None:
raise UrlError('invalid path')
ids = match.groupdict()
#print(ids)
id = '{t1}{t2}{t3}'.format(**ids)
THUMB_ROOT = self.get_config('thumb_root').rstrip('/')
SUPPORTED_SIZE = self.get_config('support_size').split(',')
org_path = '{t1}/{t2}/{t3}.{ext}'.format(**ids)
org_file = '{0}/orig/{1}'.format(THUMB_ROOT, org_path)
if not os.path.exists(org_file):
# check old id for redirect
doc = self.get_meta(ids=[id])
if doc and doc['id'] != id and 'filename' in doc:
print 'found %s' % doc['filename']
thumb_path = self.get_config('thumb_path')
new_path = '{}/{}/{}'.format(thumb_path, ids['size'], doc['filename'])
raise HttpFound('found', path=new_path)
print('fetching file: {}'.format(org_path))
file = self.fetch(id, path=org_path)
if file is None:
print('fetch failed')
raise UrlError('id {} not found'.format(id))
save_file(org_file, file)
if not os.path.exists(org_file):
raise UrlError('file not found')
# start thumbnail image
if ids['size'] == 'orig':
dst_path = 'orig/{}'.format(org_path)
dst_file = org_file
else:
dst_path = '{0}/{1}'.format(ids['size'], org_path)
dst_file = '{0}/{1}'.format(THUMB_ROOT, dst_path)
mode = ids['size'][0]
dimension = ids['size'][1:]
if dimension not in SUPPORTED_SIZE:
#print('unsupported size: {} {}'.format(mode, dimension))
raise UrlError('unsupported size')
if ids['x'] is None:
size = int(dimension)
width, height = size, size
else:
width, height = map(int, dimension.split('x'))
if not os.path.exists(dst_file):
print('start thumbnail image {} {} => {}x{}'.format(mode, dimension, width, height))
thumb_image(org_file, width, height, dst_file, mode)
if ids['mop'] == 'w' and width < 100:
raise UrlError('bad size')
if ids['mop'] is not None:
if ids['mop'] == 'w': # watermark modifier
org_file = '{}/{}/{}'.format(THUMB_ROOT, ids['size'], org_path)
dst_file = '{}/{}{}/{}'.format(THUMB_ROOT, ids['size'], ids['mop'], org_path)
if watermark_image(org_file, dst_file):
dst_path = '{}{}/{}'.format(ids['size'], ids['mop'], org_path)
else:
raise UrlError('bad modifier')
#print('dst_path: {}'.format(dst_path))
#print('dst_file: {}'.format(dst_file))
return (dst_file, dst_path)
def fetch(self, id, path):
key = path if self.engine == 's3' else id
return self._get(key)
# try:
# return self._get(key)
# except Exception, e:
# print('prepare: {} not found'.format(key))
# print e
# raise e
def url(self, path, size='orig'):
url_prefix = self.get_config('url_prefix')
thumb_path = self.get_config('thumb_path')
return '{}/{}/{}/{}'.format(url_prefix.rstrip('/'), thumb_path.strip('/'), size, path)
