def test_append_file(self): cl = Client(TEST_NS, HOSTS) key = 'test_file_%s_%s' % (random.random(), time.time()) cl.store_content(key, "SPAM") self.assertTrue(cl.get_paths(key)) self.assertEqual("SPAM", cl.get_file_data(key)) fp = cl.edit_file(key) self.assertTrue(fp) fp.seek(4) fp.write("HamEggs") fp.close() self.assertEqual("SPAMHamEggs", cl.get_file_data(key))
def main(argv): mog_domain = '' mog_trackers = '' output_dir = '' try: opts, args = getopt.getopt(argv, "d:t:o:", ["domain=", "trackers=", "output-dir="]) except getopt.GetoptError: print sys.argv[0] + ' -d <domain> -t <trackers> -o <output-dir>' for opt, arg in opts: if opt in ("-d", "--domain"): mog_domain = arg elif opt in ("-t", "--trackers"): mog_trackers = arg elif opt in ("-o", "--output-dir"): if arg[len(arg) - 1] == '/': output_dir = arg else: output_dir = arg + '/' if not os.path.exists(output_dir): os.makedirs(output_dir) datastore = Client(domain=mog_domain, trackers=mog_trackers.split(',')) keys = datastore.keys() start = time.time() for index in range(len(keys)): print('downloading: ' + keys[index]) file = open(output_dir + keys[index], 'w') file.write(datastore.get_file_data(keys[index])) file.close() print 'dump completed' print("time used : %.2f seconds" % (time.time() - start))
def test_store_content(self): client = Client(TEST_NS, HOSTS) key = 'test_file_%s_%s' % (random.random(), time.time()) data = ''.join(random.choice("0123456789") for _ in xrange(8192 * 2)) length = client.store_content(key, data) self.assertEqual(length, len(data)) content = client.get_file_data(key) self.assertEqual(content, data)
def test_new_file(): cl = Client(TEST_NS, HOSTS) key = 'test_file_%s_%s' % (random.random(), time.time()) with cl.new_file(key) as fp: assert fp.__exit__ fp.write(key) assert cl.get_paths(key) assert cl.get_file_data(key) == key
def test_store_content(): client = Client(TEST_NS, HOSTS) key = 'test_file_%s_%s' % (random.random(), time.time()) data = ''.join(random.choice("0123456789") for x in xrange(8192 * 2)) length = client.store_content(key, data) assert length == len(data) content = client.get_file_data(key) assert content == data
def test_edit_file(): # TODO # PASS return cl = Client(TEST_NS, HOSTS) key = 'test_file_%s_%s' % (random.random(), time.time()) cl.store_content(key, "SPAM") assert cl.get_paths(key) assert cl.get_file_data(key) == "SPAM" fp = cl.edit_file(key) assert fp fp.write("s") fp.seek(2) fp.write("a") fp.close() assert cl.get_file_data(key) == "sPaM"
def test_store_file(self): client = Client(TEST_NS, HOSTS) key = "test_file_%s_%s" % (random.random(), time.time()) data = "".join(random.choice("0123456789") for _ in xrange(8192 * 2)) fp = StringIO(data) length = client.store_file(key, fp) self.assertEqual(length, len(data)) content = client.get_file_data(key) self.assertEqual(content, data)
def test_seek_negative(self): client = Client(TEST_NS, HOSTS) key = 'test_file_%s_%s' % (random.random(), time.time()) fp = client.new_file(key) fp.write("SPAM") fp.seek(-10) self.assertEqual(fp.tell(), 0) fp.write("s") fp.close() self.assertEqual(client.get_file_data(key), "sPAM")
def test_seek(self): client = Client(TEST_NS, HOSTS) key = "test_file_%s_%s" % (random.random(), time.time()) fp = client.new_file(key) fp.write("SPAM") fp.seek(1) self.assertEqual(fp.tell(), 1) fp.write("p") fp.close() self.assertEqual(client.get_file_data(key), "SpAM")
def func(largefile): client = Client(TEST_NS, HOSTS) key = 'test_file_%s_%s' % (random.random(), time.time()) fp = client.new_file(key, largefile=largefile) fp.write("SPAM") fp.seek(-10) assert fp.tell() == 0 fp.write("s") fp.close() assert client.get_file_data(key) == "sPAM"
def test_rename(): client = Client(TEST_NS, HOSTS) key = 'test_file_%s_%s' % (random.random(), time.time()) client.new_file(key).write(key) paths = client.get_paths(key) assert paths newkey = 'test_file2_%s_%s' % (random.random(), time.time()) client.rename(key, newkey) paths = client.get_paths(newkey) assert paths content = client.get_file_data(newkey) assert content == key
def test_rename(self): client = Client(TEST_NS, HOSTS) key = 'test_file_%s_%s' % (random.random(), time.time()) client.new_file(key).write(key) paths = client.get_paths(key) self.assertTrue(paths) newkey = 'test_file2_%s_%s' % (random.random(), time.time()) client.rename(key, newkey) paths = client.get_paths(newkey) self.assertTrue(paths) content = client.get_file_data(newkey) assert content == key
def test_rename(self): client = Client(TEST_NS, HOSTS) key = "test_file_%s_%s" % (random.random(), time.time()) client.new_file(key).write(key) paths = client.get_paths(key) self.assertTrue(paths) newkey = "test_file2_%s_%s" % (random.random(), time.time()) client.rename(key, newkey) paths = client.get_paths(newkey) self.assertTrue(paths) content = client.get_file_data(newkey) assert content == key
def test_new_large_file(): client = Client(TEST_NS, HOSTS) key = 'test_file_%s_%s' % (random.random(), time.time()) fp = client.new_file(key, largefile=True) assert fp is not None for x in xrange(50): fp.write("0123456789") fp.close() paths = client.get_paths(key) #assert len(paths) > 1, "should exist in one ore more places" assert paths content = client.get_file_data(key) assert content == "0123456789" * 50
def test_new_file(): client = Client(TEST_NS, HOSTS) key = 'test_file_%s_%s' % (random.random(), time.time()) fp = client.new_file(key) assert fp is not None data = "0123456789" * 50 fp.write(data) fp.close() paths = client.get_paths(key) #assert len(paths) > 1, "should exist in one ore more places" assert paths content = client.get_file_data(key) assert content == data
def test_new_file(self): client = Client(TEST_NS, HOSTS) key = 'test_file_%s_%s' % (random.random(), time.time()) fp = client.new_file(key) assert fp is not None data = "0123456789" * 50 fp.write(data) fp.close() paths = client.get_paths(key) #assert len(paths) > 1, "should exist in one ore more places" assert paths content = client.get_file_data(key) assert content == data
cpub = Client(domain="publicmedia", trackers=['127.0.0.1:7001']) # list keys c.list_keys() # create a file in mogilefs f = c.new_file('foobar.txt') f.write('hi, my name bar, foo bar.') f.close() # show paths c.get_paths('foobar.txt') c.get_paths('404.txt') # get file data c.get_file_data('404.txt') c.get_file_data('foobar.txt') # remove c.delete('foobar.txt') # django storage backend from ogv_documents.storage.mogilefs import MogilefsStorage from django.core.files.base import ContentFile s = MogilefsStorage(domain='privatemedia', container='documents', base_url='/documents/', trackers=['127.0.0.1:7001']) s.exists('six/2011/justone.pdf') s.save(u'epic.txt', ContentFile('my epic content. now with a content file'))
class Storage(BaseStorage): def __init__(self, context): BaseStorage.__init__(self, context) domain = self.context.config.MOGILEFS_STORAGE_DOMAIN trackers = self.context.config.MOGILEFS_STORAGE_TRACKERS self.storage = Client(domain=domain, trackers=trackers) def __key_for(self, url): return 'thumbor-crypto-%s' % url def __detector_key_for(self, url): return 'thumbor-detector-%s' % url def put(self, path, bytes): fp = self.storage.new_file(path) fp.write(bytes) fp.close() return path def put_crypto(self, path, data): if not self.context.config.STORES_CRYPTO_KEY_FOR_EACH_IMAGE: return if not self.context.server.security_key: raise RuntimeError( """STORES_CRYPTO_KEY_FOR_EACH_IMAGE can't be True if no SECURITY_KEY specified""") key = self.__key_for(path) fp = self.storage.new_file(key) fp.write(self.context.server.security_key) fp.close() return key def put_detector_data(self, path, data): key = self.__detector_key_for(path) fp = self.storage.new_file(key) fp.write(dumps(data)) fp.close() return key def get_crypto(self, path): if not self.context.config.STORES_CRYPTO_KEY_FOR_EACH_IMAGE: return None crypto = self.storage.get_file_data(self.__key_for(path)) if not crypto: return None return crypto def get_detector_data(self, path): data = self.storage.get_file_data(self.__detector_key_for(path)) if not data: return None return loads(data) def get(self, path): return self.storage.get_file_data(path) is not None def exists(self, path): return self.storage.keys(path) is not None def remove(self, path): if not self.exists(path): return return self.storage.delete(path)
def read_image_data_by_store_key(store_key): _datastore = Client(domain=settings.MOGILEFS_DOMAIN, trackers=settings.MOGILEFS_TRACKERS) return _datastore.get_file_data(store_key)
class Storage(BaseStorage): def __init__(self, context): BaseStorage.__init__(self, context) domain = self.context.config.MOGILEFS_STORAGE_DOMAIN trackers = self.context.config.MOGILEFS_STORAGE_TRACKERS self.storage = Client(domain=domain, trackers=trackers) def __key_for(self, url): return 'thumbor-crypto-%s' % url def __detector_key_for(self, url): return 'thumbor-detector-%s' % url def put(self, path, bytes): fp = self.storage.new_file(path) fp.write(bytes) fp.close() return path def put_crypto(self, path, data): if not self.context.config.STORES_CRYPTO_KEY_FOR_EACH_IMAGE: return if not self.context.server.security_key: raise RuntimeError("""STORES_CRYPTO_KEY_FOR_EACH_IMAGE can't be True if no SECURITY_KEY specified""") key = self.__key_for(path) fp = self.storage.new_file(key) fp.write(self.context.server.security_key) fp.close() return key def put_detector_data(self, path, data): key = self.__detector_key_for(path) fp = self.storage.new_file(key) fp.write(dumps(data)) fp.close() return key def get_crypto(self, path): if not self.context.config.STORES_CRYPTO_KEY_FOR_EACH_IMAGE: return None crypto = self.storage.get_file_data(self.__key_for(path)) if not crypto: return None return crypto def get_detector_data(self, path): data = self.storage.get_file_data(self.__detector_key_for(path)) if not data: return None return loads(data) def get(self, path): return self.storage.get_file_data(path) is not None def exists(self, path): return self.storage.keys(path) is not None def remove(self, path): if not self.exists(path): return return self.storage.delete(path)