def testDirSize(self):
fake_task_uid = 1234
# Create a directory with some files and sub-directory and check its size.
test_dir = resolver._temp_download_dir(self.get_temp_dir(), fake_task_uid)
tf_v1.gfile.MakeDirs(test_dir)
tf_utils.atomic_write_string_to_file(
os.path.join(test_dir, "file1"), "content1", False)
tf_utils.atomic_write_string_to_file(
os.path.join(test_dir, "file2"), "content2", False)
test_sub_dir = os.path.join(test_dir, "sub_dir")
tf_v1.gfile.MakeDirs(test_sub_dir)
tf_utils.atomic_write_string_to_file(
os.path.join(test_sub_dir, "file3"), "content3", False)
self.assertEqual(3 * 8, resolver._dir_size(test_dir))
self.assertEqual(8, resolver._dir_size(test_sub_dir))
# Treat the directory as a temporary directory used by a module download by
# referring to that directory from the lock file.
fake_lock_filename = resolver._lock_filename(self.get_temp_dir())
tf_utils.atomic_write_string_to_file(
fake_lock_filename, resolver._lock_file_contents(fake_task_uid), False)
self.assertEqual(3 * 8, resolver._locked_tmp_dir_size(fake_lock_filename))
# Check that if temp directory doesn't exist, 0 is returned.
tf_v1.gfile.DeleteRecursively(test_dir)
self.assertEqual(0, resolver._locked_tmp_dir_size(fake_lock_filename))
def testWaitForLockToDisappear_DownloadOngoing(self):
module_dir = os.path.join(self.get_temp_dir(), "module")
task_uid = uuid.uuid4().hex
lock_filename = resolver._lock_filename(module_dir)
lock_file_content = resolver._lock_file_contents(task_uid)
tf_utils.atomic_write_string_to_file(
lock_filename, lock_file_content, overwrite=False)
lock_expiration_wait_time_secs = 10
thread = threading.Thread(
target=resolver._wait_for_lock_to_disappear,
args=(
"module",
lock_filename,
lock_expiration_wait_time_secs,
))
thread.start()
# Simulate download by writing a file every 1 sec. While writes are happing
# the lock file remains in place.
tmp_dir = resolver._temp_download_dir(self.get_temp_dir(), task_uid)
tf_v1.gfile.MakeDirs(tmp_dir)
for x in range(2 * lock_expiration_wait_time_secs):
tf_utils.atomic_write_string_to_file(
os.path.join(tmp_dir, "file_%d" % x), "test", overwrite=False)
# While writes are happening the original lock file is in place.
self.assertEqual(lock_file_content,
tf_utils.read_file_to_string(lock_filename))
time.sleep(1)
thread.join(lock_expiration_wait_time_secs)
def testWaitForLockToDisappear_DownloadOngoing(self):
module_dir = os.path.join(self.get_temp_dir(), "module")
task_uid = uuid.uuid4().hex
lock_filename = resolver._lock_filename(module_dir)
lock_file_content = resolver._lock_file_contents(task_uid)
tf_utils.atomic_write_string_to_file(
lock_filename, lock_file_content, overwrite=False)
lock_expiration_wait_time_secs = 10
thread = threading.Thread(
target=resolver._wait_for_lock_to_disappear,
args=(
"module",
lock_filename,
lock_expiration_wait_time_secs,
))
thread.start()
# Simulate download by writing a file every 1 sec. While writes are happing
# the lock file remains in place.
tmp_dir = resolver._temp_download_dir(self.get_temp_dir(), task_uid)
tf.gfile.MakeDirs(tmp_dir)
for x in range(2 * lock_expiration_wait_time_secs):
tf_utils.atomic_write_string_to_file(
os.path.join(tmp_dir, "file_%d" % x), "test", overwrite=False)
# While writes are happening the original lock file is in place.
self.assertEqual(lock_file_content,
tf_utils.read_file_to_string(lock_filename))
time.sleep(1)
thread.join(lock_expiration_wait_time_secs)
def testDirSize(self):
fake_task_uid = 1234
# Create a directory with some files and sub-directory and check its size.
test_dir = resolver._temp_download_dir(self.get_temp_dir(), fake_task_uid)
tf.gfile.MakeDirs(test_dir)
tf_utils.atomic_write_string_to_file(
os.path.join(test_dir, "file1"), "content1", False)
tf_utils.atomic_write_string_to_file(
os.path.join(test_dir, "file2"), "content2", False)
test_sub_dir = os.path.join(test_dir, "sub_dir")
tf.gfile.MakeDirs(test_sub_dir)
tf_utils.atomic_write_string_to_file(
os.path.join(test_sub_dir, "file3"), "content3", False)
self.assertEqual(3 * 8, resolver._dir_size(test_dir))
self.assertEqual(8, resolver._dir_size(test_sub_dir))
# Treat the directory as a temporary directory used by a module download by
# referring to that directory from the lock file.
fake_lock_filename = resolver._lock_filename(self.get_temp_dir())
tf_utils.atomic_write_string_to_file(
fake_lock_filename, resolver._lock_file_contents(fake_task_uid), False)
self.assertEqual(3 * 8, resolver._locked_tmp_dir_size(fake_lock_filename))
# Check that if temp directory doesn't exist, 0 is returned.
tf.gfile.DeleteRecursively(test_dir)
self.assertEqual(0, resolver._locked_tmp_dir_size(fake_lock_filename))
def testWaitForLockToDisappear_DownloadAborted(self):
module_dir = os.path.join(self.get_temp_dir(), "module")
task_uid = uuid.uuid4().hex
lock_filename = resolver._lock_filename(module_dir)
lock_file_content = resolver._lock_file_contents(task_uid)
tf_utils.atomic_write_string_to_file(
lock_filename, lock_file_content, overwrite=False)
tmp_dir = resolver._temp_download_dir(self.get_temp_dir(), task_uid)
tf_v1.gfile.MakeDirs(tmp_dir)
thread = threading.Thread(target=resolver._wait_for_lock_to_disappear,
args=("module", lock_filename, 10,))
thread.start()
thread.join(30)
# Because nobody was writing to tmp_dir, the lock file got reclaimed by
# resolver._wait_for_lock_to_disappear.
self.assertFalse(tf_v1.gfile.Exists(lock_filename))
def testWaitForLockToDisappear_DownloadAborted(self):
module_dir = os.path.join(self.get_temp_dir(), "module")
task_uid = uuid.uuid4().hex
lock_filename = resolver._lock_filename(module_dir)
lock_file_content = resolver._lock_file_contents(task_uid)
tf_utils.atomic_write_string_to_file(
lock_filename, lock_file_content, overwrite=False)
tmp_dir = resolver._temp_download_dir(self.get_temp_dir(), task_uid)
tf.gfile.MakeDirs(tmp_dir)
thread = threading.Thread(target=resolver._wait_for_lock_to_disappear,
args=("module", lock_filename, 10,))
thread.start()
thread.join(30)
# Because nobody was writing to tmp_dir, the lock file got reclaimed by
# resolver._wait_for_lock_to_disappear.
self.assertFalse(tf.gfile.Exists(lock_filename))
def testTempDownloadDir(self):
self.assertEquals("/a/b.t.tmp", resolver._temp_download_dir("/a/b/", "t"))
def testTempDownloadDir(self):
self.assertEquals("/a/b.t.tmp", resolver._temp_download_dir("/a/b/", "t"))