class WeedFS(object): master_addr = "localhost" master_port = 9333 def __init__(self, master_addr='localhost', master_port=9333, use_session=False): '''Creates WeedFS instance. Args: **master_addr**: Address of weed-fs master server (default: localhost) **master_port**: Weed-fs master server port (default: 9333) **use_session**: Use request.Session() for connections instead of requests themselves. (default: False) Returns: WeedFS instance. ''' self.master_addr = master_addr self.master_port = master_port self.conn = Connection(use_session) def __repr__(self): return "<{0} {1}:{2}>".format( self.__class__.__name__, self.master_addr, self.master_port ) def get_file(self, fid): """Get file from WeedFS. Returns file content. May be problematic for large files as content is stored in memory. Args: **fid**: File identifier <volume_id>,<file_name_hash> Returns: Content of the file with provided fid or None if file doesn't exist on the server .. versionadded:: 0.3.1 """ url = self.get_file_url(fid) return self.conn.get_raw_data(url) def get_file_url(self, fid, public=True): """ Get url for the file :param string fid: File ID :param boolean public: public or internal url :rtype: string """ try: volume_id, rest = fid.strip().split(",") except ValueError: raise BadFidFormat( "fid must be in format: <volume_id>,<file_name_hash>") file_location = self.get_file_location(volume_id) volume_url = file_location.public_url if public else file_location.url url = "http://{volume_url}/{fid}".format( volume_url=volume_url, fid=fid) return url def get_file_location(self, volume_id): """ Get location for the file, WeedFS volume is choosed randomly :param integer volume_id: volume_id :rtype: namedtuple `FileLocation` `{"public_url":"", "url":""}` """ url = ("http://{master_addr}:{master_port}/" "dir/lookup?volumeId={volume_id}").format( master_addr=self.master_addr, master_port=self.master_port, volume_id=volume_id) data = json.loads(self.conn.get_data(url)) _file_location = random.choice(data['locations']) FileLocation = namedtuple('FileLocation', "public_url url") return FileLocation(_file_location['publicUrl'], _file_location['url']) def get_file_size(self, fid): """ Gets size of uploaded file Or None if file doesn't exist. Args: **fid**: File identifier <volume_id>,<file_name_hash> Returns: Int or None """ url = self.get_file_url(fid) res = self.conn.head(url) if res is not None: size = res.headers.get("content-length", None) if size is not None: return int(size) return None def file_exists(self, fid): """Checks if file with provided fid exists Args: **fid**: File identifier <volume_id>,<file_name_hash> Returns: True if file exists. False if not. """ res = self.get_file_size(fid) if res is not None: return True return False def delete_file(self, fid): """ Delete file from WeedFS :param string fid: File ID """ url = self.get_file_url(fid) return self.conn.delete_data(url) def upload_file(self, path=None, stream=None, name=None, **kwargs): """ Uploads file to WeedFS I takes either path or stream and name and upload it to WeedFS server. Returns fid of the uploaded file. :param string path: :param string stream: :param string name: :rtype: string or None """ params = "&".join(["%s=%s" % (k, v) for k, v in kwargs.items()]) url = "http://{master_addr}:{master_port}/dir/assign{params}".format( master_addr=self.master_addr, master_port=self.master_port, params="?" + params if params else '' ) data = json.loads(self.conn.get_data(url)) if data.get("error") is not None: return None post_url = "http://{url}/{fid}".format(**data) if path is not None: filename = os.path.basename(path) with open(path, "rb") as file_stream: res = self.conn.post_file(post_url, filename, file_stream) # we have file like object and filename elif stream is not None and name is not None: res = self.conn.post_file(post_url, name, stream) else: raise ValueError( "If `path` is None then *both* `stream` and `name` must not" " be None ") response_data = json.loads(res) if "size" in response_data: return data.get('fid') return None def vacuum(self, threshold=0.3): ''' Force garbage collection :param float threshold (optional): The threshold is optional, and will not change the default threshold. :rtype: boolean ''' url = ("http://{master_addr}:{master_port}/" "vol/vacuum?garbageThreshold={threshold}").format( master_addr=self.master_addr, master_port=self.master_port, threshold=threshold) res = self.conn.get_data(url) if res is not None: return True return False @property def version(self): ''' Returns Weed-FS master version :rtype: string ''' url = "http://{master_addr}:{master_port}/dir/status".format( master_addr=self.master_addr, master_port=self.master_port) data = self.conn.get_data(url) response_data = json.loads(data) return response_data.get("Version")
class WeedFS(object): master_addr = "localhost" master_port = 9333 def __init__(self, master_addr='localhost', master_port=9333, use_session=False, use_public_url=True): '''Creates WeedFS instance. Args: **master_addr**: Address of weed-fs master server (default: localhost) **master_port**: Weed-fs master server port (default: 9333) **use_session**: Use request.Session() for connections instead of requests themselves. (default: False) **use_public_url**: If ``True``, all the requests will use ``publicUrl`` link instead of ``url``. Returns: WeedFS instance. ''' self.master_addr = master_addr self.master_port = master_port self.conn = Connection(use_session) self.use_public_url = use_public_url def __repr__(self): return "<{0} {1}:{2}>".format( self.__class__.__name__, self.master_addr, self.master_port ) def get_file(self, fid): """Get file from WeedFS. Returns file content. May be problematic for large files as content is stored in memory. Args: **fid**: File identifier <volume_id>,<file_name_hash> Returns: Content of the file with provided fid or None if file doesn't exist on the server .. versionadded:: 0.3.1 """ url = self.get_file_url(fid) return self.conn.get_raw_data(url) def get_file_url(self, fid, public=None): """ Get url for the file :param string fid: File ID :param boolean public: public or internal url :rtype: string """ try: volume_id, rest = fid.strip().split(",") except ValueError: raise BadFidFormat( "fid must be in format: <volume_id>,<file_name_hash>") file_location = self.get_file_location(volume_id) if public is None: public = self.use_public_url volume_url = file_location.public_url if public else file_location.url url = "http://{volume_url}/{fid}".format( volume_url=volume_url, fid=fid) return url def get_file_location(self, volume_id): """ Get location for the file, WeedFS volume is choosed randomly :param integer volume_id: volume_id :rtype: namedtuple `FileLocation` `{"public_url":"", "url":""}` """ url = ("http://{master_addr}:{master_port}/" "dir/lookup?volumeId={volume_id}").format( master_addr=self.master_addr, master_port=self.master_port, volume_id=volume_id) data = json.loads(self.conn.get_data(url)) _file_location = random.choice(data['locations']) FileLocation = namedtuple('FileLocation', "public_url url") return FileLocation(_file_location['publicUrl'], _file_location['url']) def get_file_size(self, fid): """ Gets size of uploaded file Or None if file doesn't exist. Args: **fid**: File identifier <volume_id>,<file_name_hash> Returns: Int or None """ url = self.get_file_url(fid) res = self.conn.head(url) if res is not None: size = res.headers.get("content-length", None) if size is not None: return int(size) return None def file_exists(self, fid): """Checks if file with provided fid exists Args: **fid**: File identifier <volume_id>,<file_name_hash> Returns: True if file exists. False if not. """ res = self.get_file_size(fid) if res is not None: return True return False def delete_file(self, fid): """ Delete file from WeedFS :param string fid: File ID """ url = self.get_file_url(fid) return self.conn.delete_data(url) def upload_file(self, path=None, stream=None, name=None, **kwargs): """ Uploads file to WeedFS I takes either path or stream and name and upload it to WeedFS server. Returns fid of the uploaded file. :param string path: :param string stream: :param string name: :rtype: string or None """ params = "&".join(["%s=%s" % (k, v) for k, v in kwargs.items()]) url = "http://{master_addr}:{master_port}/dir/assign{params}".format( master_addr=self.master_addr, master_port=self.master_port, params="?" + params if params else '' ) data = json.loads(self.conn.get_data(url)) if data.get("error") is not None: return None post_url = "http://{url}/{fid}".format( url=data['publicUrl' if self.use_public_url else 'url'], fid=data['fid'] ) if path is not None: filename = os.path.basename(path) with open(path, "rb") as file_stream: res = self.conn.post_file(post_url, filename, file_stream) # we have file like object and filename elif stream is not None and name is not None: res = self.conn.post_file(post_url, name, stream) else: raise ValueError( "If `path` is None then *both* `stream` and `name` must not" " be None ") response_data = json.loads(res) if "size" in response_data: return data.get('fid') return None def vacuum(self, threshold=0.3): ''' Force garbage collection :param float threshold (optional): The threshold is optional, and will not change the default threshold. :rtype: boolean ''' url = ("http://{master_addr}:{master_port}/" "vol/vacuum?garbageThreshold={threshold}").format( master_addr=self.master_addr, master_port=self.master_port, threshold=threshold) res = self.conn.get_data(url) if res is not None: return True return False @property def version(self): ''' Returns Weed-FS master version :rtype: string ''' url = "http://{master_addr}:{master_port}/dir/status".format( master_addr=self.master_addr, master_port=self.master_port) data = self.conn.get_data(url) response_data = json.loads(data) return response_data.get("Version")
class ReqTests(unittest.TestCase): def setUp(self): self.conn = Connection() pass def test_post_file(self): with HTTMock(response_content): r = self.conn.post_file("http://utek.pl", "tests.py", open(__file__, "rb")) self.assertEqual(r, "OK") with HTTMock(response_content_201): r = self.conn.post_file("http://utek.pl", "tests.py", open(__file__, "rb")) self.assertEqual(r, "OK") with HTTMock(response_content_404): r = self.conn.post_file("http://utek.pl", "tests.py", open(__file__, "rb")) self.assertIsNone(r) def test_get_data(self): with HTTMock(response_content): r = self.conn.get_data("http://utek.pl") self.assertEqual(r, "OK") with HTTMock(response_content_404): r = self.conn.get_data("http://utek.pl") self.assertIsNone(r) def test_get_raw_data(self): with HTTMock(response_content): r = self.conn.get_raw_data("http://utek.pl") self.assertEqual(r, b"OK") with HTTMock(response_content_404): r = self.conn.get_raw_data("http://utek.pl") self.assertIsNone(r) def test_delete_data(self): with HTTMock(response_content): r = self.conn.delete_data("http://localhost") self.assertTrue(r) with HTTMock(response_content_202): r = self.conn.delete_data("http://localhost") self.assertTrue(r) with HTTMock(response_content_404): r = self.conn.delete_data("http://localhost") self.assertFalse(r) def test_prepare_headers(self): headers = self.conn._prepare_headers() self.assertIsInstance(headers, dict) for k, v in headers.items(): self.assertIsInstance(k, str) self.assertIsInstance(v, str) def test_additional_headers(self): additional_headers = {"X-Test": "123"} kwargs = {"additional_headers": additional_headers} headers = self.conn._prepare_headers(**kwargs) self.assertIsInstance(headers, dict) self.assertIsNotNone(headers.get("X-Test")) with HTTMock(response_content): r = self.conn.post_file( "http://utek.pl", "tests.py", open(__file__, "rb"), additional_headers=additional_headers, ) self.assertEqual(r, "OK") r = self.conn.get_data("http://utek.pl", additional_headers=additional_headers) self.assertEqual(r, "OK") r = self.conn.get_raw_data("http://utek.pl", additional_headers=additional_headers) self.assertEqual(r, b"OK") r = self.conn.delete_data("http://localhost", additional_headers=additional_headers) self.assertTrue(r)
class ReqTests(unittest.TestCase): def setUp(self): self.conn = Connection() pass def test_post_file(self): with HTTMock(response_content): r = self.conn.post_file("http://utek.pl", "tets.py", open(__file__, "rb")) self.assertEqual(r, "OK") with HTTMock(response_content_201): r = self.conn.post_file("http://utek.pl", "tets.py", open(__file__, "rb")) self.assertEqual(r, "OK") with HTTMock(response_content_404): r = self.conn.post_file("http://utek.pl", "tets.py", open(__file__, "rb")) self.assertIsNone(r) def test_get_data(self): with HTTMock(response_content): r = self.conn.get_data("http://utek.pl") self.assertEqual(r, "OK") with HTTMock(response_content_404): r = self.conn.get_data("http://utek.pl") self.assertIsNone(r) def test_get_raw_data(self): with HTTMock(response_content): r = self.conn.get_raw_data("http://utek.pl") self.assertEqual(r, b"OK") with HTTMock(response_content_404): r = self.conn.get_raw_data("http://utek.pl") self.assertIsNone(r) def test_delete_data(self): with HTTMock(response_content): r = self.conn.delete_data("http://localhost") self.assertTrue(r) with HTTMock(response_content_202): r = self.conn.delete_data("http://localhost") self.assertTrue(r) with HTTMock(response_content_404): r = self.conn.delete_data("http://localhost") self.assertFalse(r) def test_prepare_headers(self): headers = self.conn._prepare_headers() self.assertIsInstance(headers, dict) for k, v in headers.items(): self.assertIsInstance(k, str) self.assertIsInstance(v, str) def test_additional_headers(self): additional_headers = {"X-Test": "123"} kwargs = {"additional_headers": additional_headers} headers = self.conn._prepare_headers(**kwargs) self.assertIsInstance(headers, dict) self.assertIsNotNone(headers.get("X-Test")) with HTTMock(response_content): r = self.conn.post_file("http://utek.pl", "tets.py", open(__file__, "rb"), additional_headers=additional_headers) self.assertEqual(r, "OK") r = self.conn.get_data("http://utek.pl", additional_headers=additional_headers) self.assertEqual(r, "OK") r = self.conn.get_raw_data("http://utek.pl", additional_headers=additional_headers) self.assertEqual(r, b"OK") r = self.conn.delete_data("http://localhost", additional_headers=additional_headers) self.assertTrue(r)