def test_upload_dir_with_last_time(self, client): dirname1 = tempfile.mkdtemp() fpath1 = dirname1 + "/test1.txt" with open(fpath1, "w") as f: f.write("data1") fpath2 = dirname1 + "/test2.txt" with open(fpath2, "w") as f: f.write("data2") last_time = to_datetime(os.stat(fpath2).st_mtime) time.sleep(0.1) dirname2 = tempfile.mkdtemp(prefix=dirname1 + "/") fpath3 = dirname2 + "/test3.txt" with open(fpath3, "w") as f: f.write("data3") store = AzureBlobStoreService() store.set_connection(connection=client) blob_path = "path/to/" azure_url = self.wasbs_base + blob_path rel_path1 = dirname1.split("/")[-1] rel_path2 = dirname2.split("/")[-1] # Test without basename store.upload_dir(dirname=dirname1, blob=azure_url, use_basename=False, last_time=last_time) call_args_list = client.get_container_client( ).upload_blob.call_args_list for call_args in call_args_list: call_arg1, call_arg2 = call_args[0] if call_arg1 == "{}{}/test3.txt".format(blob_path, rel_path2): assert call_arg2.name == fpath3 else: assert False # Test with basename store.upload_dir(dirname=dirname1, blob=azure_url, use_basename=True, last_time=last_time) call_args_list = client.get_container_client( ).upload_blob.call_args_list[1:] for call_args in call_args_list: call_arg1, call_arg2 = call_args[0] if call_arg1 == "{}{}/{}/test3.txt".format(blob_path, rel_path1, rel_path2): assert call_arg2.name == fpath3 else: assert False
def test_upload_dir_with_last_time(self): dirname1 = tempfile.mkdtemp() fpath1 = dirname1 + "/test1.txt" with open(fpath1, "w") as f: f.write("data1") fpath2 = dirname1 + "/test2.txt" with open(fpath2, "w") as f: f.write("data2") last_time = to_datetime(os.stat(fpath2).st_mtime) time.sleep(0.1) dirname2 = tempfile.mkdtemp(prefix=dirname1 + "/") fpath3 = dirname2 + "/test3.txt" with open(fpath3, "w") as f: f.write("data3") store = LocalStore() path_to = tempfile.mkdtemp() rel_path1 = dirname1.split("/")[-1] rel_path2 = dirname2.split("/")[-1] # Test without basename assert os.path.exists(os.path.join(path_to, "test1.txt")) is False assert os.path.exists(os.path.join(path_to, "test2.txt")) is False assert os.path.exists(os.path.join(path_to, rel_path2, "test3.txt")) is False store.upload_dir(dirname=dirname1, path_to=path_to, use_basename=False, last_time=last_time) assert os.path.exists(os.path.join(path_to, "test1.txt")) is False assert os.path.exists(os.path.join(path_to, "test2.txt")) is False assert os.path.exists(os.path.join(path_to, rel_path2, "test3.txt")) is True # Test with basename path_to = tempfile.mkdtemp() assert os.path.exists(os.path.join(path_to, rel_path1, "test1.txt")) is False assert os.path.exists(os.path.join(path_to, rel_path1, "test2.txt")) is False assert (os.path.exists( os.path.join(path_to, rel_path1, rel_path2, "test3.txt")) is False) store.upload_dir(dirname=dirname1, path_to=path_to, use_basename=True, last_time=last_time) assert os.path.exists(os.path.join(path_to, "test1.txt")) is False assert os.path.exists(os.path.join(path_to, "test2.txt")) is False assert (os.path.exists( os.path.join(path_to, rel_path1, rel_path2, "test3.txt")) is True)
def format_record(self, record): message = "" if record.msg: message = record.msg % record.args return V1Log.process_log_line( value=message, timestamp=to_datetime(record.created), node=self._node, pod=self._pod, container=self._container, )
def test_upload_dir_with_last_time(self, client): dirname1 = tempfile.mkdtemp() fpath1 = dirname1 + "/test1.txt" with open(fpath1, "w") as f: f.write("data1") fpath2 = dirname1 + "/test2.txt" with open(fpath2, "w") as f: f.write("data2") last_time = to_datetime(os.stat(fpath2).st_mtime) time.sleep(0.1) dirname2 = tempfile.mkdtemp(prefix=dirname1 + "/") fpath3 = dirname2 + "/test3.txt" with open(fpath3, "w") as f: f.write("data3") store = AzureBlobStoreService() blob_path = "path/to/" azure_url = self.wasbs_base + blob_path rel_path1 = dirname1.split("/")[-1] rel_path2 = dirname2.split("/")[-1] # Test without basename store.upload_dir( dirname=dirname1, blob=azure_url, use_basename=False, last_time=last_time ) client.return_value.create_blob_from_path.assert_has_calls( [ mock.call( "container", "{}{}/test3.txt".format(blob_path, rel_path2), fpath3 ) ], any_order=True, ) # Test with basename store.upload_dir( dirname=dirname1, blob=azure_url, use_basename=True, last_time=last_time ) client.return_value.create_blob_from_path.assert_has_calls( [ mock.call( "container", "{}{}/{}/test3.txt".format(blob_path, rel_path1, rel_path2), fpath3, ) ], any_order=True, )
def test_upload_directory_with_last_time(self): store = S3Service() store.connection.create_bucket(Bucket="bucket") dirname1 = tempfile.mkdtemp() fpath1 = dirname1 + "/test1.txt" with open(fpath1, "w") as f: f.write("data1") fpath2 = dirname1 + "/test2.txt" with open(fpath2, "w") as f: f.write("data2") last_time = to_datetime(os.stat(fpath2).st_mtime) time.sleep(0.1) dirname2 = tempfile.mkdtemp(prefix=dirname1 + "/") fpath3 = dirname2 + "/test3.txt" with open(fpath3, "w") as f: f.write("data3") rel_path1 = dirname1.split("/")[-1] rel_path2 = dirname2.split("/")[-1] # Test without using basename # Upload store.upload_dir(dirname1, "mykey", "bucket", use_basename=False, last_time=last_time) assert store.check_key("mykey/test1.txt", "bucket") is False assert store.check_key("mykey/test2.txt", "bucket") is False assert store.check_key("mykey/{}/test3.txt".format(rel_path2), "bucket") is True # Test with using basename store.upload_dir(dirname1, "mykey", "bucket", use_basename=True, last_time=last_time) assert (store.check_key("mykey/{}/test1.txt".format(rel_path1), "bucket") is False) assert (store.check_key("mykey/{}/test2.txt".format(rel_path1), "bucket") is False) assert (store.check_key( "mykey/{}/{}/test3.txt".format(rel_path1, rel_path2), "bucket") is True)