def test_append_file(self):
cl = Client(TEST_NS, HOSTS)
key = 'test_file_%s_%s' % (random.random(), time.time())
cl.store_content(key, "SPAM")
self.assertTrue(cl.get_paths(key))
self.assertEqual("SPAM", cl.get_file_data(key))
fp = cl.edit_file(key)
self.assertTrue(fp)
fp.seek(4)
fp.write("HamEggs")
fp.close()
self.assertEqual("SPAMHamEggs", cl.get_file_data(key))
def main(argv):
mog_domain = ''
mog_trackers = ''
output_dir = ''
try:
opts, args = getopt.getopt(argv, "d:t:o:",
["domain=", "trackers=", "output-dir="])
except getopt.GetoptError:
print sys.argv[0] + ' -d <domain> -t <trackers> -o <output-dir>'
for opt, arg in opts:
if opt in ("-d", "--domain"):
mog_domain = arg
elif opt in ("-t", "--trackers"):
mog_trackers = arg
elif opt in ("-o", "--output-dir"):
if arg[len(arg) - 1] == '/':
output_dir = arg
else:
output_dir = arg + '/'
if not os.path.exists(output_dir):
os.makedirs(output_dir)
datastore = Client(domain=mog_domain, trackers=mog_trackers.split(','))
keys = datastore.keys()
start = time.time()
for index in range(len(keys)):
print('downloading: ' + keys[index])
file = open(output_dir + keys[index], 'w')
file.write(datastore.get_file_data(keys[index]))
file.close()
print 'dump completed'
print("time used : %.2f seconds" % (time.time() - start))
def test_store_content(self):
client = Client(TEST_NS, HOSTS)
key = 'test_file_%s_%s' % (random.random(), time.time())
data = ''.join(random.choice("0123456789") for _ in xrange(8192 * 2))
length = client.store_content(key, data)
self.assertEqual(length, len(data))
content = client.get_file_data(key)
self.assertEqual(content, data)
def test_new_file():
cl = Client(TEST_NS, HOSTS)
key = 'test_file_%s_%s' % (random.random(), time.time())
with cl.new_file(key) as fp:
assert fp.__exit__
fp.write(key)
assert cl.get_paths(key)
assert cl.get_file_data(key) == key
def test_store_content():
client = Client(TEST_NS, HOSTS)
key = 'test_file_%s_%s' % (random.random(), time.time())
data = ''.join(random.choice("0123456789") for x in xrange(8192 * 2))
length = client.store_content(key, data)
assert length == len(data)
content = client.get_file_data(key)
assert content == data
def test_store_content(self):
client = Client(TEST_NS, HOSTS)
key = 'test_file_%s_%s' % (random.random(), time.time())
data = ''.join(random.choice("0123456789") for _ in xrange(8192 * 2))
length = client.store_content(key, data)
self.assertEqual(length, len(data))
content = client.get_file_data(key)
self.assertEqual(content, data)
def test_edit_file():
# TODO
# PASS
return
cl = Client(TEST_NS, HOSTS)
key = 'test_file_%s_%s' % (random.random(), time.time())
cl.store_content(key, "SPAM")
assert cl.get_paths(key)
assert cl.get_file_data(key) == "SPAM"
fp = cl.edit_file(key)
assert fp
fp.write("s")
fp.seek(2)
fp.write("a")
fp.close()
assert cl.get_file_data(key) == "sPaM"
def test_store_file(self):
client = Client(TEST_NS, HOSTS)
key = "test_file_%s_%s" % (random.random(), time.time())
data = "".join(random.choice("0123456789") for _ in xrange(8192 * 2))
fp = StringIO(data)
length = client.store_file(key, fp)
self.assertEqual(length, len(data))
content = client.get_file_data(key)
self.assertEqual(content, data)
def test_seek_negative(self):
client = Client(TEST_NS, HOSTS)
key = 'test_file_%s_%s' % (random.random(), time.time())
fp = client.new_file(key)
fp.write("SPAM")
fp.seek(-10)
self.assertEqual(fp.tell(), 0)
fp.write("s")
fp.close()
self.assertEqual(client.get_file_data(key), "sPAM")
def test_seek_negative(self):
client = Client(TEST_NS, HOSTS)
key = 'test_file_%s_%s' % (random.random(), time.time())
fp = client.new_file(key)
fp.write("SPAM")
fp.seek(-10)
self.assertEqual(fp.tell(), 0)
fp.write("s")
fp.close()
self.assertEqual(client.get_file_data(key), "sPAM")
def test_seek(self):
client = Client(TEST_NS, HOSTS)
key = "test_file_%s_%s" % (random.random(), time.time())
fp = client.new_file(key)
fp.write("SPAM")
fp.seek(1)
self.assertEqual(fp.tell(), 1)
fp.write("p")
fp.close()
self.assertEqual(client.get_file_data(key), "SpAM")
def func(largefile):
client = Client(TEST_NS, HOSTS)
key = 'test_file_%s_%s' % (random.random(), time.time())
fp = client.new_file(key, largefile=largefile)
fp.write("SPAM")
fp.seek(-10)
assert fp.tell() == 0
fp.write("s")
fp.close()
assert client.get_file_data(key) == "sPAM"
def test_rename():
client = Client(TEST_NS, HOSTS)
key = 'test_file_%s_%s' % (random.random(), time.time())
client.new_file(key).write(key)
paths = client.get_paths(key)
assert paths
newkey = 'test_file2_%s_%s' % (random.random(), time.time())
client.rename(key, newkey)
paths = client.get_paths(newkey)
assert paths
content = client.get_file_data(newkey)
assert content == key
def test_rename(self):
client = Client(TEST_NS, HOSTS)
key = 'test_file_%s_%s' % (random.random(), time.time())
client.new_file(key).write(key)
paths = client.get_paths(key)
self.assertTrue(paths)
newkey = 'test_file2_%s_%s' % (random.random(), time.time())
client.rename(key, newkey)
paths = client.get_paths(newkey)
self.assertTrue(paths)
content = client.get_file_data(newkey)
assert content == key
def test_rename(self):
client = Client(TEST_NS, HOSTS)
key = "test_file_%s_%s" % (random.random(), time.time())
client.new_file(key).write(key)
paths = client.get_paths(key)
self.assertTrue(paths)
newkey = "test_file2_%s_%s" % (random.random(), time.time())
client.rename(key, newkey)
paths = client.get_paths(newkey)
self.assertTrue(paths)
content = client.get_file_data(newkey)
assert content == key
def test_new_large_file():
client = Client(TEST_NS, HOSTS)
key = 'test_file_%s_%s' % (random.random(), time.time())
fp = client.new_file(key, largefile=True)
assert fp is not None
for x in xrange(50):
fp.write("0123456789")
fp.close()
paths = client.get_paths(key)
#assert len(paths) > 1, "should exist in one ore more places"
assert paths
content = client.get_file_data(key)
assert content == "0123456789" * 50
def test_new_file():
client = Client(TEST_NS, HOSTS)
key = 'test_file_%s_%s' % (random.random(), time.time())
fp = client.new_file(key)
assert fp is not None
data = "0123456789" * 50
fp.write(data)
fp.close()
paths = client.get_paths(key)
#assert len(paths) > 1, "should exist in one ore more places"
assert paths
content = client.get_file_data(key)
assert content == data
def test_new_file(self):
client = Client(TEST_NS, HOSTS)
key = 'test_file_%s_%s' % (random.random(), time.time())
fp = client.new_file(key)
assert fp is not None
data = "0123456789" * 50
fp.write(data)
fp.close()
paths = client.get_paths(key)
#assert len(paths) > 1, "should exist in one ore more places"
assert paths
content = client.get_file_data(key)
assert content == data
cpub = Client(domain="publicmedia", trackers=['127.0.0.1:7001'])
# list keys
c.list_keys()
# create a file in mogilefs
f = c.new_file('foobar.txt')
f.write('hi, my name bar, foo bar.')
f.close()
# show paths
c.get_paths('foobar.txt')
c.get_paths('404.txt')
# get file data
c.get_file_data('404.txt')
c.get_file_data('foobar.txt')
# remove
c.delete('foobar.txt')
# django storage backend
from ogv_documents.storage.mogilefs import MogilefsStorage
from django.core.files.base import ContentFile
s = MogilefsStorage(domain='privatemedia', container='documents', base_url='/documents/', trackers=['127.0.0.1:7001'])
s.exists('six/2011/justone.pdf')
s.save(u'epic.txt', ContentFile('my epic content. now with a content file'))
class Storage(BaseStorage):
def __init__(self, context):
BaseStorage.__init__(self, context)
domain = self.context.config.MOGILEFS_STORAGE_DOMAIN
trackers = self.context.config.MOGILEFS_STORAGE_TRACKERS
self.storage = Client(domain=domain, trackers=trackers)
def __key_for(self, url):
return 'thumbor-crypto-%s' % url
def __detector_key_for(self, url):
return 'thumbor-detector-%s' % url
def put(self, path, bytes):
fp = self.storage.new_file(path)
fp.write(bytes)
fp.close()
return path
def put_crypto(self, path, data):
if not self.context.config.STORES_CRYPTO_KEY_FOR_EACH_IMAGE:
return
if not self.context.server.security_key:
raise RuntimeError(
"""STORES_CRYPTO_KEY_FOR_EACH_IMAGE can't be True
if no SECURITY_KEY specified""")
key = self.__key_for(path)
fp = self.storage.new_file(key)
fp.write(self.context.server.security_key)
fp.close()
return key
def put_detector_data(self, path, data):
key = self.__detector_key_for(path)
fp = self.storage.new_file(key)
fp.write(dumps(data))
fp.close()
return key
def get_crypto(self, path):
if not self.context.config.STORES_CRYPTO_KEY_FOR_EACH_IMAGE:
return None
crypto = self.storage.get_file_data(self.__key_for(path))
if not crypto:
return None
return crypto
def get_detector_data(self, path):
data = self.storage.get_file_data(self.__detector_key_for(path))
if not data:
return None
return loads(data)
def get(self, path):
return self.storage.get_file_data(path) is not None
def exists(self, path):
return self.storage.keys(path) is not None
def remove(self, path):
if not self.exists(path):
return
return self.storage.delete(path)
def read_image_data_by_store_key(store_key):
_datastore = Client(domain=settings.MOGILEFS_DOMAIN,
trackers=settings.MOGILEFS_TRACKERS)
return _datastore.get_file_data(store_key)
class Storage(BaseStorage):
def __init__(self, context):
BaseStorage.__init__(self, context)
domain = self.context.config.MOGILEFS_STORAGE_DOMAIN
trackers = self.context.config.MOGILEFS_STORAGE_TRACKERS
self.storage = Client(domain=domain, trackers=trackers)
def __key_for(self, url):
return 'thumbor-crypto-%s' % url
def __detector_key_for(self, url):
return 'thumbor-detector-%s' % url
def put(self, path, bytes):
fp = self.storage.new_file(path)
fp.write(bytes)
fp.close()
return path
def put_crypto(self, path, data):
if not self.context.config.STORES_CRYPTO_KEY_FOR_EACH_IMAGE:
return
if not self.context.server.security_key:
raise RuntimeError("""STORES_CRYPTO_KEY_FOR_EACH_IMAGE can't be True
if no SECURITY_KEY specified""")
key = self.__key_for(path)
fp = self.storage.new_file(key)
fp.write(self.context.server.security_key)
fp.close()
return key
def put_detector_data(self, path, data):
key = self.__detector_key_for(path)
fp = self.storage.new_file(key)
fp.write(dumps(data))
fp.close()
return key
def get_crypto(self, path):
if not self.context.config.STORES_CRYPTO_KEY_FOR_EACH_IMAGE:
return None
crypto = self.storage.get_file_data(self.__key_for(path))
if not crypto:
return None
return crypto
def get_detector_data(self, path):
data = self.storage.get_file_data(self.__detector_key_for(path))
if not data:
return None
return loads(data)
def get(self, path):
return self.storage.get_file_data(path) is not None
def exists(self, path):
return self.storage.keys(path) is not None
def remove(self, path):
if not self.exists(path):
return
return self.storage.delete(path)
