def __init__(self,
base_url=None,
container=None,
domain=None,
trackers=None,
client=None):
"""
:param base_url: public facing url prefix. e.g "/my/private/docs/"
:param container: prefix to use for the mogilefs storage. e.g "documents"
:param domain: mogilefs domain to store stuff into. e.g "privatemedia"
:param trackers: list of mogilefs trackers
:return:
with the above examples the file "my/file.pdf" will be handled like this:
mogilefs:
domain: privatemedia
key: documents/my/file.pdf
public url: /my/private/docs/my/file.pdf
"""
self.base_url = base_url or settings.MEDIA_URL
self.container = container or getattr(settings,
'MOGILEFS_MEDIA_CONTAINER', '')
self.domain = domain or getattr(settings, 'MOGILEFS_DOMAIN', '')
self.trackers = trackers or getattr(settings, 'MOGILEFS_TRACKERS', [])
self.client = client or Client(domain=self.domain,
trackers=self.trackers)
def test_file_like_object(self):
client = Client(TEST_NS, HOSTS)
key = 'test_file_%s_%s' % (random.random(), time.time())
fp = client.new_file(key)
fp.write("spam\negg\nham\n")
fp.seek(0)
line = fp.readline()
self.assertEqual(line, "spam\n")
line = fp.readline()
self.assertEqual(line, "egg\n")
line = fp.readline()
self.assertEqual(line, "ham\n")
line = fp.readline()
self.assertEqual(line, '')
fp.seek(0)
lines = fp.readlines()
self.assertEqual(lines, ["spam\n", "egg\n", "ham\n"])
fp.close()
def update(self, title = None, group_id = None, image_data = None, status = None):
self.__ensure_category_obj()
if title != None:
self.category_obj.title = title
if group_id != None:
self.category_obj.group_id = group_id
if status != None:
self.category_obj.status = int(status)
if image_data != None:
self.__datastore = Client(
domain = settings.MOGILEFS_DOMAIN,
trackers = settings.MOGILEFS_TRACKERS
)
_key = md5(image_data).hexdigest()
_large_data = self.__resize(image_data, 86, 86)
_fp = self.__datastore.new_file('category/large/' + _key)
_fp.write(_large_data)
_fp.close()
_small_data = self.__resize(image_data, 43, 43)
_fp = self.__datastore.new_file('category/small/' + _key)
_fp.write(_small_data)
_fp.close()
self.category_obj.image_store_hash = _key
self.category_obj.save()
def photo_thumbnail(request, owner, nb):
storage = Client(domain=settings.MOGILEFS_DOMAIN,
trackers=settings.MOGILEFS_TRACKERS)
f = storage.read_file(
path.join(settings.PROCESSED_PHOTOS_DIR,
'%s,%s-160x160.jpg' % (owner, nb)))
return HttpResponse(make_reader(f), content_type='image/jpeg')
def main(argv):
mog_domain = ''
mog_trackers = ''
output_dir = ''
try:
opts, args = getopt.getopt(argv, "d:t:o:",
["domain=", "trackers=", "output-dir="])
except getopt.GetoptError:
print sys.argv[0] + ' -d <domain> -t <trackers> -o <output-dir>'
for opt, arg in opts:
if opt in ("-d", "--domain"):
mog_domain = arg
elif opt in ("-t", "--trackers"):
mog_trackers = arg
elif opt in ("-o", "--output-dir"):
if arg[len(arg) - 1] == '/':
output_dir = arg
else:
output_dir = arg + '/'
if not os.path.exists(output_dir):
os.makedirs(output_dir)
datastore = Client(domain=mog_domain, trackers=mog_trackers.split(','))
keys = datastore.keys()
start = time.time()
for index in range(len(keys)):
print('downloading: ' + keys[index])
file = open(output_dir + keys[index], 'w')
file.write(datastore.get_file_data(keys[index]))
file.close()
print 'dump completed'
print("time used : %.2f seconds" % (time.time() - start))
def test_mkcol(self):
client = Client(TEST_NS, HOSTS)
for x in xrange(0, 10):
key = 'test_file_%s_%s_%d' % (random.random(), time.time(), x)
client.new_file(key).write("SPAM%s" % x)
paths = client.get_paths(key)
self.assertTrue(paths)
def test_read_file(self):
client = Client(TEST_NS, HOSTS)
key = 'test_file_%s_%s' % (random.random(), time.time())
client.store_content(key, key)
fp = client.read_file(key)
self.assertNotEqual(fp, None)
self.assertEqual(key, fp.read())
def __init__(self, location=None, base_url=None, **kwargs):
self.clients = {}
for server in settings.DISTRIBUTED_MOGILEFS_CONFIG['SERVERS']:
srv = settings.DISTRIBUTED_MOGILEFS_CONFIG['SERVERS'][server]
self.clients[server] = Client(domain=srv['DOMAIN'],
trackers=srv['TRACKERS'])
self.servers = settings.DISTRIBUTED_MOGILEFS_CONFIG['SERVERS']
self.ring = HashRing(settings.DISTRIBUTED_MOGILEFS_CONFIG['SLOTS'])
self.kwargs = kwargs
def test_rename_dupliate_key(self):
client = Client(TEST_NS, HOSTS)
key1 = 'test_file_%s_%s' % (random.random(), time.time())
key2 = 'key2:' + key1
client.store_content(key1, key1)
client.store_content(key2, key2)
self.assertEqual(client.rename(key1, key2), False)
def test_store_content(self):
client = Client(TEST_NS, HOSTS)
key = 'test_file_%s_%s' % (random.random(), time.time())
data = ''.join(random.choice("0123456789") for _ in xrange(8192 * 2))
length = client.store_content(key, data)
self.assertEqual(length, len(data))
content = client.get_file_data(key)
self.assertEqual(content, data)
def test_closed_file(self):
client = Client(TEST_NS, HOSTS)
key = 'test_file_%s_%s' % (random.random(), time.time())
fp = client.new_file(key)
fp.write("spam")
fp.close()
self.assertRaises(ValueError, fp.write, "egg")
self.assertRaises(ValueError, fp.read)
self.assertRaises(ValueError, fp.seek, 0)
self.assertRaises(ValueError, fp.tell)
def test_delete(self):
client = Client(TEST_NS, HOSTS)
key = 'test_file_%s_%s' % (random.random(), time.time())
client.new_file(key).write("SPAM")
paths = client.get_paths(key)
self.assertTrue(paths)
client.delete(key)
paths = client.get_paths(key)
self.assertFalse(paths)
def test_seek_negative(self):
client = Client(TEST_NS, HOSTS)
key = 'test_file_%s_%s' % (random.random(), time.time())
fp = client.new_file(key)
fp.write("SPAM")
fp.seek(-10)
self.assertEqual(fp.tell(), 0)
fp.write("s")
fp.close()
self.assertEqual(client.get_file_data(key), "sPAM")
def test_seek_read(self):
client = Client(TEST_NS, HOSTS)
key = 'test_file_%s_%s' % (random.random(), time.time())
client.store_content(key, "0123456789")
fp = client.read_file(key)
fp.seek(1)
self.assertEqual(fp.tell(), 1)
content = fp.read(3)
assert content == "123"
self.assertEqual(fp.tell(), 4)
def test_rename(self):
client = Client(TEST_NS, HOSTS)
key = 'test_file_%s_%s' % (random.random(), time.time())
client.new_file(key).write(key)
paths = client.get_paths(key)
self.assertTrue(paths)
newkey = 'test_file2_%s_%s' % (random.random(), time.time())
client.rename(key, newkey)
paths = client.get_paths(newkey)
self.assertTrue(paths)
content = client.get_file_data(newkey)
assert content == key
def get_mogile_client(self, node):
''' Initialize Mogilefs client against given node:port combination '''
logging.debug(
'Initializing mogilefs client for domain: %s, tracker: %s',
self.domain, node)
client = Client(domain=self.domain, trackers=node.split(','))
# Test if we can execute simple request on mogilefs server
if client.sleep(1):
logging.info('Successfuly connected to mogilefs server %s', node)
return client
else:
raise Exception(
'Connection to mogilefs server {} failed'.format(node))
def test_new_file(self):
client = Client(TEST_NS, HOSTS)
key = 'test_file_%s_%s' % (random.random(), time.time())
fp = client.new_file(key)
assert fp is not None
data = "0123456789" * 50
fp.write(data)
fp.close()
paths = client.get_paths(key)
#assert len(paths) > 1, "should exist in one ore more places"
assert paths
content = client.get_file_data(key)
assert content == data
def prepare_photo_files(owner, nb):
storage = Client(domain=settings.MOGILEFS_DOMAIN,
trackers=settings.MOGILEFS_TRACKERS)
#im = Image.open(path.join(settings.UNPROCESSED_PHOTOS_DIR, '%s.jpg' % id))
f = storage.read_file(
path.join(settings.UNPROCESSED_PHOTOS_DIR, '%s,%s.jpg' % (owner, nb)))
im = Image.open(f)
im.thumbnail((160, 160), Image.ANTIALIAS)
tn = storage.new_file(
path.join(settings.PROCESSED_PHOTOS_DIR,
'%s,%s-160x160.jpg' % (owner, nb)))
im.save(tn, 'JPEG')
#sleep(20)
photo = Photo.get(owner, nb)
photo.ready()
def create(cls, user_id, origin_data):
_key = md5(origin_data).hexdigest()
_datastore = Client(
domain = settings.MOGILEFS_DOMAIN,
trackers = settings.MOGILEFS_TRACKERS
)
_origin_link = 'avatar/origin/' + _key
_fp = _datastore.new_file(_origin_link)
_fp.write(origin_data)
_fp.close()
_square_data = cls.crop_square(origin_data)
_large_link = 'avatar/large/' + _key
_large_data = cls.resize(_square_data, 180, 180)
_fp = _datastore.new_file(_large_link)
_fp.write(_large_data)
_fp.close()
_small_link = 'avatar/small/' + _key
_small_data = cls.resize(_square_data, 50, 50)
_fp = _datastore.new_file(_small_link)
_fp.write(_small_data)
_fp.close()
_inst = cls(user_id)
if _inst.avatar_obj == None:
_avatar_obj = AvatarModel.objects.create(
user_id = user_id,
avatar_origin = _origin_link,
avatar_small = _small_link,
avatar_large = _large_link
)
_inst.avatar_obj = _avatar_obj
else:
_inst.avatar_obj.avatar_origin = _origin_link
_inst.avatar_obj.avatar_large = _large_link
_inst.avatar_obj.avatar_small = _small_link
_inst.avatar_obj.save()
return _inst
def upload(request):
storage = Client(domain=settings.MOGILEFS_DOMAIN,
trackers=settings.MOGILEFS_TRACKERS)
photo_file = request.FILES.get('photo')
photo_desc = request.POST.get('desc', '')
if photo_file is None or not photo_file.name.endswith('.jpg') or \
photo_file.size > settings.MAX_PHOTO_SIZE:
# Mamy błąd
return HttpResponseRedirect('/')
else:
model = Photo.create(owner=request.user.id, desc=photo_desc)
#with open(path.join(settings.UNPROCESSED_PHOTOS_DIR, '%s.jpg' % model.id), 'wb') as f:
with storage.new_file(
path.join(settings.UNPROCESSED_PHOTOS_DIR,
'%s,%s.jpg' % (model.owner, model.nb))) as f:
for chunk in photo_file.chunks():
f.write(chunk)
f.close()
prepare_photo_files.delay(model.owner, model.nb)
return HttpResponseRedirect('/')
def save_square_image_data_fixed(cls, store_hash, image_data):
# _image_sizes = [64, 128, 240, 310, 480, 640]
log.info(store_hash)
_datastore = Client(domain=settings.MOGILEFS_DOMAIN,
trackers=settings.MOGILEFS_TRACKERS)
# _square_data = cls.crop_square(image_data)
# _img = WandImage(blob = image_data)
# if _img.width > 800 or _img.height > 800:
# _square_data = cls.resize(image_data, 800, 800)
_fp = _datastore.new_file('img/' + store_hash + '.jpg')
_fp.write(image_data)
_fp.close()
for _size in Images_size:
log.info(_size)
_data_resized = cls.resize(image_data, _size, _size)
_fp = _datastore.new_file('img/' + store_hash + '.jpg_' +
str(_size) + 'x' + str(_size) + '.jpg')
_fp.write(_data_resized)
_fp.close()
def test_list_keys(self):
keys = ["spam", "egg", "ham"]
domain = "test:list_keys:%s:%s:%s" % (random.random(), time.time(),
TEST_NS)
self.moga.create_domain(domain)
mogc = Client(domain, HOSTS)
for k in keys:
mogc.store_content(k, k)
try:
files = mogc.list_keys()
assert len(files) == 3
files = mogc.list_keys(limit=1)
assert len(files) == 1
files = mogc.list_keys(prefix='sp')
assert len(files) == 1
finally:
for k in keys:
mogc.delete(k)
self.moga.delete_domain(domain)
#-*- coding: utf-8 -*-
from pymogile import Client
c = Client(domain="privatemedia", trackers=['127.0.0.1:7001'])
cpub = Client(domain="publicmedia", trackers=['127.0.0.1:7001'])
# list keys
c.list_keys()
# create a file in mogilefs
f = c.new_file('foobar.txt')
f.write('hi, my name bar, foo bar.')
f.close()
# show paths
c.get_paths('foobar.txt')
c.get_paths('404.txt')
# get file data
c.get_file_data('404.txt')
c.get_file_data('foobar.txt')
# remove
c.delete('foobar.txt')
# django storage backend
from ogv_documents.storage.mogilefs import MogilefsStorage
from django.core.files.base import ContentFile
def __ensure_datastore_client(self):
if not hasattr(self, '__entity_obj'):
self.__datastore = Client(domain=settings.MOGILEFS_DOMAIN,
trackers=settings.MOGILEFS_TRACKERS)
def read_image_data_by_store_key(store_key):
_datastore = Client(domain=settings.MOGILEFS_DOMAIN,
trackers=settings.MOGILEFS_TRACKERS)
return _datastore.get_file_data(store_key)
def test_new_file_unexisting_domain(self):
client = Client('unexisting_domain', HOSTS)
key = 'test_file_%s_%s' % (random.random(), time.time())
self.assertRaises(MogileFSError, client.new_file, key)
def test_new_file_unexisting_class(self):
client = Client(TEST_NS, HOSTS)
key = 'test_file_%s_%s' % (random.random(), time.time())
self.assertRaises(MogileFSError, client.new_file, key, 'unexisting')
title = title.replace(' ', '')
if not categories.has_key(title):
print title
else:
categories[title]['large'] = f
else:
title = f.replace('.png', '')
title = title.replace(' ', '/')
if not categories.has_key(title):
print title
else:
categories[title]['small'] = f
from pymogile import Client
from hashlib import md5
datastore = Client(domain='staging', trackers=['10.0.1.23:7001'])
for title, obj in categories.items():
if obj.has_key('large') and obj.has_key('small'):
f = open(image_path + '/' + obj['large'], 'r')
large_data = f.read()
f.close()
key = md5(large_data).hexdigest()
f = open(image_path + '/' + obj['small'], 'r')
small_data = f.read()
f.close()
mgf = datastore.new_file('category/large/' + key)
mgf.write(large_data)
mgf.close()
def test_sleep(self):
client = Client(TEST_NS, HOSTS)
self.assertEqual(client.sleep(1), True)
def save_origin_image_data(cls, store_hash, image_data):
_datastore = Client(domain=settings.MOGILEFS_DOMAIN,
trackers=settings.MOGILEFS_TRACKERS)
_fp = _datastore.new_file('img/' + store_hash + '.jpg')
_fp.write(image_data)
_fp.close()
