def download(self, url) -> (IO, int): r = requests.get(url, stream=True, verify=self.verify) log.debug(func=log.ensure_json_serialization, url=url, reponse_headers=r.headers) return r.raw, int(r.headers['Content-length'])
def test_debug_with_erasure(self): d = { "id": "741fa660-3880-416e-a959-11111111111", "timestamp": 1587723588670, "client": "dstack-py", "version": "0.3.0", "os": { "sysname": "Darwin", "release": "19.3.0", "version": "Darwin Kernel Version 19.3.0: Thu Jan 9 20:58:23 PST 2020;", "machine": "x86_64" }, "message": "my message", "attachments": [{ "data": "test data", "type": "image/svg", "description": "My first plot", "params": { "x": 10, "y": 20 } }, { "data": "test data 2", "type": "image/svg", "description": "My second plot", "params": { "x": 100, "y": 200 } }], "stack": "user/my_stack" } my_logger = log.InMemoryLogger() # test default level ERASE_BINARY_DATA | ERASE_PARAM_VALUES log.enable(logger=my_logger) log.debug(func=log.erase_sensitive_data, data=d, extra="extra") d1 = json.loads(my_logger.io.getvalue()) data = d1["data"] for i in [0, 1]: self.assertTrue( data["attachments"][i]["data"].startswith("erased")) self.assertTrue( data["attachments"][i]["params"]["x"].startswith("erased")) self.assertTrue( data["attachments"][i]["params"]["y"].startswith("erased")) self.assertEqual("extra", d1["extra"])
def test_log_to_file(self): file = Path(tempfile.gettempdir()) / "dstack.log" my_logger = log.FileLogger(str(file.resolve())) log.enable(logger=my_logger) log.debug(message="test message") log.debug(message="second message") with open(file.resolve(), "r") as f: lines = f.readlines() d0 = json.loads(lines[0]) d1 = json.loads(lines[1]) self.assertEqual("test message", d0["message"]) self.assertEqual("second message", d1["message"])
def test_log_to_file(self): filename = "/tmp/test/dstack.log" if Path(filename).exists(): os.remove(filename) my_logger = log.FileLogger(filename) log.enable(logger=my_logger) log.debug(message="test message") log.debug(message="second message") with open(filename, "r") as f: lines = f.readlines() d0 = json.loads(lines[0]) d1 = json.loads(lines[1]) self.assertEqual("test message", d0["message"]) self.assertEqual("second message", d1["message"])
def test_simple_log(self): my_logger = log.InMemoryLogger() log.enable(logger=my_logger) self.assertTrue(log.debug(data={"test": 123}, url="api/test")) self.assertTrue(log.debug(data="hello", x=10, y=20)) lines = my_logger.io.getvalue().splitlines() self.assertEqual(2, len(lines)) d = json.loads(lines[0]) self.assertTrue("time" in d) self.assertEqual(123, d["data"]["test"]) self.assertEqual("api/test", d["url"]) d = json.loads(lines[1]) self.assertTrue("time" in d) self.assertEqual(10, d["x"]) self.assertEqual(20, d["y"]) log.disable() self.assertFalse(log.debug(test="test"))
def do_request(self, endpoint: str, data: Optional[Dict], token: Optional[str], method: str = "POST", stack: Optional[str] = None) -> Dict: url = self.url + endpoint event_id = log.uuid() log.debug(event_id=event_id, func=log.erase_sensitive_data, url=url, method=method, data=data) headers = {} if token is not None: headers["Authorization"] = f"Bearer {token}" if data is None: response = requests.request(method=method, url=url, headers=headers, verify=self.verify) else: data_bytes = json.dumps(data).encode(self.ENCODING) headers[ "Content-Type"] = f"application/json; charset={self.ENCODING}" response = requests.request(method=method, url=url, data=data_bytes, headers=headers, verify=self.verify) log.debug(event_id=event_id, func=log.erase_token, request_headers=response.request.headers) log.debug(event_id=event_id, func=log.ensure_json_serialization, response_headers=response.headers) if response.status_code != 200: # FIXME: parse content log.debug(event_id=event_id, response_body=str(response.content)) if stack and response.status_code == 404: raise StackNotFoundError(stack) response.raise_for_status() return response.json()
def do_upload(self, upload_url: str, data: Content): event_id = log.uuid() log.debug(event_id=event_id, url=upload_url, length=data.length()) response = requests.put(url=upload_url, data=data.stream(), verify=self.verify) log.debug(event_id=event_id, func=log.ensure_json_serialization, request_headers=response.request.headers) log.debug(event_id=event_id, func=log.ensure_json_serialization, response_headers=response.headers) response.raise_for_status()
def test_none(self): my_logger = log.InMemoryLogger() log.enable(logger=my_logger) log.debug(data=None, extra=None)