def main(argv): mog_domain = '' mog_trackers = '' output_dir = '' try: opts, args = getopt.getopt(argv, "d:t:o:", ["domain=", "trackers=", "output-dir="]) except getopt.GetoptError: print sys.argv[0] + ' -d <domain> -t <trackers> -o <output-dir>' for opt, arg in opts: if opt in ("-d", "--domain"): mog_domain = arg elif opt in ("-t", "--trackers"): mog_trackers = arg elif opt in ("-o", "--output-dir"): if arg[len(arg) - 1] == '/': output_dir = arg else: output_dir = arg + '/' if not os.path.exists(output_dir): os.makedirs(output_dir) datastore = Client(domain=mog_domain, trackers=mog_trackers.split(',')) keys = datastore.keys() start = time.time() for index in range(len(keys)): print('downloading: ' + keys[index]) file = open(output_dir + keys[index], 'w') file.write(datastore.get_file_data(keys[index])) file.close() print 'dump completed' print("time used : %.2f seconds" % (time.time() - start))
class Storage(BaseStorage): def __init__(self, context): BaseStorage.__init__(self, context) domain = self.context.config.MOGILEFS_STORAGE_DOMAIN trackers = self.context.config.MOGILEFS_STORAGE_TRACKERS self.storage = Client(domain=domain, trackers=trackers) def __key_for(self, url): return 'thumbor-crypto-%s' % url def __detector_key_for(self, url): return 'thumbor-detector-%s' % url def put(self, path, bytes): fp = self.storage.new_file(path) fp.write(bytes) fp.close() return path def put_crypto(self, path, data): if not self.context.config.STORES_CRYPTO_KEY_FOR_EACH_IMAGE: return if not self.context.server.security_key: raise RuntimeError("""STORES_CRYPTO_KEY_FOR_EACH_IMAGE can't be True if no SECURITY_KEY specified""") key = self.__key_for(path) fp = self.storage.new_file(key) fp.write(self.context.server.security_key) fp.close() return key def put_detector_data(self, path, data): key = self.__detector_key_for(path) fp = self.storage.new_file(key) fp.write(dumps(data)) fp.close() return key def get_crypto(self, path): if not self.context.config.STORES_CRYPTO_KEY_FOR_EACH_IMAGE: return None crypto = self.storage.get_file_data(self.__key_for(path)) if not crypto: return None return crypto def get_detector_data(self, path): data = self.storage.get_file_data(self.__detector_key_for(path)) if not data: return None return loads(data) def get(self, path): return self.storage.get_file_data(path) is not None def exists(self, path): return self.storage.keys(path) is not None def remove(self, path): if not self.exists(path): return return self.storage.delete(path)
class Storage(BaseStorage): def __init__(self, context): BaseStorage.__init__(self, context) domain = self.context.config.MOGILEFS_STORAGE_DOMAIN trackers = self.context.config.MOGILEFS_STORAGE_TRACKERS self.storage = Client(domain=domain, trackers=trackers) def __key_for(self, url): return 'thumbor-crypto-%s' % url def __detector_key_for(self, url): return 'thumbor-detector-%s' % url def put(self, path, bytes): fp = self.storage.new_file(path) fp.write(bytes) fp.close() return path def put_crypto(self, path, data): if not self.context.config.STORES_CRYPTO_KEY_FOR_EACH_IMAGE: return if not self.context.server.security_key: raise RuntimeError( """STORES_CRYPTO_KEY_FOR_EACH_IMAGE can't be True if no SECURITY_KEY specified""") key = self.__key_for(path) fp = self.storage.new_file(key) fp.write(self.context.server.security_key) fp.close() return key def put_detector_data(self, path, data): key = self.__detector_key_for(path) fp = self.storage.new_file(key) fp.write(dumps(data)) fp.close() return key def get_crypto(self, path): if not self.context.config.STORES_CRYPTO_KEY_FOR_EACH_IMAGE: return None crypto = self.storage.get_file_data(self.__key_for(path)) if not crypto: return None return crypto def get_detector_data(self, path): data = self.storage.get_file_data(self.__detector_key_for(path)) if not data: return None return loads(data) def get(self, path): return self.storage.get_file_data(path) is not None def exists(self, path): return self.storage.keys(path) is not None def remove(self, path): if not self.exists(path): return return self.storage.delete(path)