def download(self, url) -> (IO, int):
r = requests.get(url, stream=True, verify=self.verify)
log.debug(func=log.ensure_json_serialization,
url=url,
reponse_headers=r.headers)
return r.raw, int(r.headers['Content-length'])
def test_debug_with_erasure(self):
d = {
"id":
"741fa660-3880-416e-a959-11111111111",
"timestamp":
1587723588670,
"client":
"dstack-py",
"version":
"0.3.0",
"os": {
"sysname": "Darwin",
"release": "19.3.0",
"version":
"Darwin Kernel Version 19.3.0: Thu Jan 9 20:58:23 PST 2020;",
"machine": "x86_64"
},
"message":
"my message",
"attachments": [{
"data": "test data",
"type": "image/svg",
"description": "My first plot",
"params": {
"x": 10,
"y": 20
}
}, {
"data": "test data 2",
"type": "image/svg",
"description": "My second plot",
"params": {
"x": 100,
"y": 200
}
}],
"stack":
"user/my_stack"
}
my_logger = log.InMemoryLogger()
# test default level ERASE_BINARY_DATA | ERASE_PARAM_VALUES
log.enable(logger=my_logger)
log.debug(func=log.erase_sensitive_data, data=d, extra="extra")
d1 = json.loads(my_logger.io.getvalue())
data = d1["data"]
for i in [0, 1]:
self.assertTrue(
data["attachments"][i]["data"].startswith("erased"))
self.assertTrue(
data["attachments"][i]["params"]["x"].startswith("erased"))
self.assertTrue(
data["attachments"][i]["params"]["y"].startswith("erased"))
self.assertEqual("extra", d1["extra"])
def test_log_to_file(self):
file = Path(tempfile.gettempdir()) / "dstack.log"
my_logger = log.FileLogger(str(file.resolve()))
log.enable(logger=my_logger)
log.debug(message="test message")
log.debug(message="second message")
with open(file.resolve(), "r") as f:
lines = f.readlines()
d0 = json.loads(lines[0])
d1 = json.loads(lines[1])
self.assertEqual("test message", d0["message"])
self.assertEqual("second message", d1["message"])
def test_log_to_file(self):
filename = "/tmp/test/dstack.log"
if Path(filename).exists():
os.remove(filename)
my_logger = log.FileLogger(filename)
log.enable(logger=my_logger)
log.debug(message="test message")
log.debug(message="second message")
with open(filename, "r") as f:
lines = f.readlines()
d0 = json.loads(lines[0])
d1 = json.loads(lines[1])
self.assertEqual("test message", d0["message"])
self.assertEqual("second message", d1["message"])
def test_simple_log(self):
my_logger = log.InMemoryLogger()
log.enable(logger=my_logger)
self.assertTrue(log.debug(data={"test": 123}, url="api/test"))
self.assertTrue(log.debug(data="hello", x=10, y=20))
lines = my_logger.io.getvalue().splitlines()
self.assertEqual(2, len(lines))
d = json.loads(lines[0])
self.assertTrue("time" in d)
self.assertEqual(123, d["data"]["test"])
self.assertEqual("api/test", d["url"])
d = json.loads(lines[1])
self.assertTrue("time" in d)
self.assertEqual(10, d["x"])
self.assertEqual(20, d["y"])
log.disable()
self.assertFalse(log.debug(test="test"))
def do_request(self,
endpoint: str,
data: Optional[Dict],
token: Optional[str],
method: str = "POST",
stack: Optional[str] = None) -> Dict:
url = self.url + endpoint
event_id = log.uuid()
log.debug(event_id=event_id,
func=log.erase_sensitive_data,
url=url,
method=method,
data=data)
headers = {}
if token is not None:
headers["Authorization"] = f"Bearer {token}"
if data is None:
response = requests.request(method=method,
url=url,
headers=headers,
verify=self.verify)
else:
data_bytes = json.dumps(data).encode(self.ENCODING)
headers[
"Content-Type"] = f"application/json; charset={self.ENCODING}"
response = requests.request(method=method,
url=url,
data=data_bytes,
headers=headers,
verify=self.verify)
log.debug(event_id=event_id,
func=log.erase_token,
request_headers=response.request.headers)
log.debug(event_id=event_id,
func=log.ensure_json_serialization,
response_headers=response.headers)
if response.status_code != 200:
# FIXME: parse content
log.debug(event_id=event_id, response_body=str(response.content))
if stack and response.status_code == 404:
raise StackNotFoundError(stack)
response.raise_for_status()
return response.json()
def do_upload(self, upload_url: str, data: Content):
event_id = log.uuid()
log.debug(event_id=event_id, url=upload_url, length=data.length())
response = requests.put(url=upload_url,
data=data.stream(),
verify=self.verify)
log.debug(event_id=event_id,
func=log.ensure_json_serialization,
request_headers=response.request.headers)
log.debug(event_id=event_id,
func=log.ensure_json_serialization,
response_headers=response.headers)
response.raise_for_status()
def test_none(self):
my_logger = log.InMemoryLogger()
log.enable(logger=my_logger)
log.debug(data=None, extra=None)
