def test_upload_dir_with_last_time(self, client):
dirname1 = tempfile.mkdtemp()
fpath1 = dirname1 + "/test1.txt"
with open(fpath1, "w") as f:
f.write("data1")
fpath2 = dirname1 + "/test2.txt"
with open(fpath2, "w") as f:
f.write("data2")
last_time = to_datetime(os.stat(fpath2).st_mtime)
time.sleep(0.1)
dirname2 = tempfile.mkdtemp(prefix=dirname1 + "/")
fpath3 = dirname2 + "/test3.txt"
with open(fpath3, "w") as f:
f.write("data3")
store = AzureBlobStoreService()
store.set_connection(connection=client)
blob_path = "path/to/"
azure_url = self.wasbs_base + blob_path
rel_path1 = dirname1.split("/")[-1]
rel_path2 = dirname2.split("/")[-1]
# Test without basename
store.upload_dir(dirname=dirname1,
blob=azure_url,
use_basename=False,
last_time=last_time)
call_args_list = client.get_container_client(
).upload_blob.call_args_list
for call_args in call_args_list:
call_arg1, call_arg2 = call_args[0]
if call_arg1 == "{}{}/test3.txt".format(blob_path, rel_path2):
assert call_arg2.name == fpath3
else:
assert False
# Test with basename
store.upload_dir(dirname=dirname1,
blob=azure_url,
use_basename=True,
last_time=last_time)
call_args_list = client.get_container_client(
).upload_blob.call_args_list[1:]
for call_args in call_args_list:
call_arg1, call_arg2 = call_args[0]
if call_arg1 == "{}{}/{}/test3.txt".format(blob_path, rel_path1,
rel_path2):
assert call_arg2.name == fpath3
else:
assert False
def test_upload_dir_with_last_time(self):
dirname1 = tempfile.mkdtemp()
fpath1 = dirname1 + "/test1.txt"
with open(fpath1, "w") as f:
f.write("data1")
fpath2 = dirname1 + "/test2.txt"
with open(fpath2, "w") as f:
f.write("data2")
last_time = to_datetime(os.stat(fpath2).st_mtime)
time.sleep(0.1)
dirname2 = tempfile.mkdtemp(prefix=dirname1 + "/")
fpath3 = dirname2 + "/test3.txt"
with open(fpath3, "w") as f:
f.write("data3")
store = LocalStore()
path_to = tempfile.mkdtemp()
rel_path1 = dirname1.split("/")[-1]
rel_path2 = dirname2.split("/")[-1]
# Test without basename
assert os.path.exists(os.path.join(path_to, "test1.txt")) is False
assert os.path.exists(os.path.join(path_to, "test2.txt")) is False
assert os.path.exists(os.path.join(path_to, rel_path2,
"test3.txt")) is False
store.upload_dir(dirname=dirname1,
path_to=path_to,
use_basename=False,
last_time=last_time)
assert os.path.exists(os.path.join(path_to, "test1.txt")) is False
assert os.path.exists(os.path.join(path_to, "test2.txt")) is False
assert os.path.exists(os.path.join(path_to, rel_path2,
"test3.txt")) is True
# Test with basename
path_to = tempfile.mkdtemp()
assert os.path.exists(os.path.join(path_to, rel_path1,
"test1.txt")) is False
assert os.path.exists(os.path.join(path_to, rel_path1,
"test2.txt")) is False
assert (os.path.exists(
os.path.join(path_to, rel_path1, rel_path2, "test3.txt")) is False)
store.upload_dir(dirname=dirname1,
path_to=path_to,
use_basename=True,
last_time=last_time)
assert os.path.exists(os.path.join(path_to, "test1.txt")) is False
assert os.path.exists(os.path.join(path_to, "test2.txt")) is False
assert (os.path.exists(
os.path.join(path_to, rel_path1, rel_path2, "test3.txt")) is True)
def format_record(self, record):
message = ""
if record.msg:
message = record.msg % record.args
return V1Log.process_log_line(
value=message,
timestamp=to_datetime(record.created),
node=self._node,
pod=self._pod,
container=self._container,
)
def test_upload_dir_with_last_time(self, client):
dirname1 = tempfile.mkdtemp()
fpath1 = dirname1 + "/test1.txt"
with open(fpath1, "w") as f:
f.write("data1")
fpath2 = dirname1 + "/test2.txt"
with open(fpath2, "w") as f:
f.write("data2")
last_time = to_datetime(os.stat(fpath2).st_mtime)
time.sleep(0.1)
dirname2 = tempfile.mkdtemp(prefix=dirname1 + "/")
fpath3 = dirname2 + "/test3.txt"
with open(fpath3, "w") as f:
f.write("data3")
store = AzureBlobStoreService()
blob_path = "path/to/"
azure_url = self.wasbs_base + blob_path
rel_path1 = dirname1.split("/")[-1]
rel_path2 = dirname2.split("/")[-1]
# Test without basename
store.upload_dir(
dirname=dirname1, blob=azure_url, use_basename=False, last_time=last_time
)
client.return_value.create_blob_from_path.assert_has_calls(
[
mock.call(
"container", "{}{}/test3.txt".format(blob_path, rel_path2), fpath3
)
],
any_order=True,
)
# Test with basename
store.upload_dir(
dirname=dirname1, blob=azure_url, use_basename=True, last_time=last_time
)
client.return_value.create_blob_from_path.assert_has_calls(
[
mock.call(
"container",
"{}{}/{}/test3.txt".format(blob_path, rel_path1, rel_path2),
fpath3,
)
],
any_order=True,
)
def test_upload_directory_with_last_time(self):
store = S3Service()
store.connection.create_bucket(Bucket="bucket")
dirname1 = tempfile.mkdtemp()
fpath1 = dirname1 + "/test1.txt"
with open(fpath1, "w") as f:
f.write("data1")
fpath2 = dirname1 + "/test2.txt"
with open(fpath2, "w") as f:
f.write("data2")
last_time = to_datetime(os.stat(fpath2).st_mtime)
time.sleep(0.1)
dirname2 = tempfile.mkdtemp(prefix=dirname1 + "/")
fpath3 = dirname2 + "/test3.txt"
with open(fpath3, "w") as f:
f.write("data3")
rel_path1 = dirname1.split("/")[-1]
rel_path2 = dirname2.split("/")[-1]
# Test without using basename
# Upload
store.upload_dir(dirname1,
"mykey",
"bucket",
use_basename=False,
last_time=last_time)
assert store.check_key("mykey/test1.txt", "bucket") is False
assert store.check_key("mykey/test2.txt", "bucket") is False
assert store.check_key("mykey/{}/test3.txt".format(rel_path2),
"bucket") is True
# Test with using basename
store.upload_dir(dirname1,
"mykey",
"bucket",
use_basename=True,
last_time=last_time)
assert (store.check_key("mykey/{}/test1.txt".format(rel_path1),
"bucket") is False)
assert (store.check_key("mykey/{}/test2.txt".format(rel_path1),
"bucket") is False)
assert (store.check_key(
"mykey/{}/{}/test3.txt".format(rel_path1, rel_path2), "bucket") is
True)
