def testDirSize(self): fake_task_uid = 1234 # Create a directory with some files and sub-directory and check its size. test_dir = resolver._temp_download_dir(self.get_temp_dir(), fake_task_uid) tf_v1.gfile.MakeDirs(test_dir) tf_utils.atomic_write_string_to_file( os.path.join(test_dir, "file1"), "content1", False) tf_utils.atomic_write_string_to_file( os.path.join(test_dir, "file2"), "content2", False) test_sub_dir = os.path.join(test_dir, "sub_dir") tf_v1.gfile.MakeDirs(test_sub_dir) tf_utils.atomic_write_string_to_file( os.path.join(test_sub_dir, "file3"), "content3", False) self.assertEqual(3 * 8, resolver._dir_size(test_dir)) self.assertEqual(8, resolver._dir_size(test_sub_dir)) # Treat the directory as a temporary directory used by a module download by # referring to that directory from the lock file. fake_lock_filename = resolver._lock_filename(self.get_temp_dir()) tf_utils.atomic_write_string_to_file( fake_lock_filename, resolver._lock_file_contents(fake_task_uid), False) self.assertEqual(3 * 8, resolver._locked_tmp_dir_size(fake_lock_filename)) # Check that if temp directory doesn't exist, 0 is returned. tf_v1.gfile.DeleteRecursively(test_dir) self.assertEqual(0, resolver._locked_tmp_dir_size(fake_lock_filename))
def testWaitForLockToDisappear_DownloadOngoing(self): module_dir = os.path.join(self.get_temp_dir(), "module") task_uid = uuid.uuid4().hex lock_filename = resolver._lock_filename(module_dir) lock_file_content = resolver._lock_file_contents(task_uid) tf_utils.atomic_write_string_to_file( lock_filename, lock_file_content, overwrite=False) lock_expiration_wait_time_secs = 10 thread = threading.Thread( target=resolver._wait_for_lock_to_disappear, args=( "module", lock_filename, lock_expiration_wait_time_secs, )) thread.start() # Simulate download by writing a file every 1 sec. While writes are happing # the lock file remains in place. tmp_dir = resolver._temp_download_dir(self.get_temp_dir(), task_uid) tf_v1.gfile.MakeDirs(tmp_dir) for x in range(2 * lock_expiration_wait_time_secs): tf_utils.atomic_write_string_to_file( os.path.join(tmp_dir, "file_%d" % x), "test", overwrite=False) # While writes are happening the original lock file is in place. self.assertEqual(lock_file_content, tf_utils.read_file_to_string(lock_filename)) time.sleep(1) thread.join(lock_expiration_wait_time_secs)
def testWaitForLockToDisappear_DownloadOngoing(self): module_dir = os.path.join(self.get_temp_dir(), "module") task_uid = uuid.uuid4().hex lock_filename = resolver._lock_filename(module_dir) lock_file_content = resolver._lock_file_contents(task_uid) tf_utils.atomic_write_string_to_file( lock_filename, lock_file_content, overwrite=False) lock_expiration_wait_time_secs = 10 thread = threading.Thread( target=resolver._wait_for_lock_to_disappear, args=( "module", lock_filename, lock_expiration_wait_time_secs, )) thread.start() # Simulate download by writing a file every 1 sec. While writes are happing # the lock file remains in place. tmp_dir = resolver._temp_download_dir(self.get_temp_dir(), task_uid) tf.gfile.MakeDirs(tmp_dir) for x in range(2 * lock_expiration_wait_time_secs): tf_utils.atomic_write_string_to_file( os.path.join(tmp_dir, "file_%d" % x), "test", overwrite=False) # While writes are happening the original lock file is in place. self.assertEqual(lock_file_content, tf_utils.read_file_to_string(lock_filename)) time.sleep(1) thread.join(lock_expiration_wait_time_secs)
def testDirSize(self): fake_task_uid = 1234 # Create a directory with some files and sub-directory and check its size. test_dir = resolver._temp_download_dir(self.get_temp_dir(), fake_task_uid) tf.gfile.MakeDirs(test_dir) tf_utils.atomic_write_string_to_file( os.path.join(test_dir, "file1"), "content1", False) tf_utils.atomic_write_string_to_file( os.path.join(test_dir, "file2"), "content2", False) test_sub_dir = os.path.join(test_dir, "sub_dir") tf.gfile.MakeDirs(test_sub_dir) tf_utils.atomic_write_string_to_file( os.path.join(test_sub_dir, "file3"), "content3", False) self.assertEqual(3 * 8, resolver._dir_size(test_dir)) self.assertEqual(8, resolver._dir_size(test_sub_dir)) # Treat the directory as a temporary directory used by a module download by # referring to that directory from the lock file. fake_lock_filename = resolver._lock_filename(self.get_temp_dir()) tf_utils.atomic_write_string_to_file( fake_lock_filename, resolver._lock_file_contents(fake_task_uid), False) self.assertEqual(3 * 8, resolver._locked_tmp_dir_size(fake_lock_filename)) # Check that if temp directory doesn't exist, 0 is returned. tf.gfile.DeleteRecursively(test_dir) self.assertEqual(0, resolver._locked_tmp_dir_size(fake_lock_filename))
def testWaitForLockToDisappear_DownloadAborted(self): module_dir = os.path.join(self.get_temp_dir(), "module") task_uid = uuid.uuid4().hex lock_filename = resolver._lock_filename(module_dir) lock_file_content = resolver._lock_file_contents(task_uid) tf_utils.atomic_write_string_to_file( lock_filename, lock_file_content, overwrite=False) tmp_dir = resolver._temp_download_dir(self.get_temp_dir(), task_uid) tf_v1.gfile.MakeDirs(tmp_dir) thread = threading.Thread(target=resolver._wait_for_lock_to_disappear, args=("module", lock_filename, 10,)) thread.start() thread.join(30) # Because nobody was writing to tmp_dir, the lock file got reclaimed by # resolver._wait_for_lock_to_disappear. self.assertFalse(tf_v1.gfile.Exists(lock_filename))
def testWaitForLockToDisappear_DownloadAborted(self): module_dir = os.path.join(self.get_temp_dir(), "module") task_uid = uuid.uuid4().hex lock_filename = resolver._lock_filename(module_dir) lock_file_content = resolver._lock_file_contents(task_uid) tf_utils.atomic_write_string_to_file( lock_filename, lock_file_content, overwrite=False) tmp_dir = resolver._temp_download_dir(self.get_temp_dir(), task_uid) tf.gfile.MakeDirs(tmp_dir) thread = threading.Thread(target=resolver._wait_for_lock_to_disappear, args=("module", lock_filename, 10,)) thread.start() thread.join(30) # Because nobody was writing to tmp_dir, the lock file got reclaimed by # resolver._wait_for_lock_to_disappear. self.assertFalse(tf.gfile.Exists(lock_filename))
def testTempDownloadDir(self): self.assertEquals("/a/b.t.tmp", resolver._temp_download_dir("/a/b/", "t"))
def testTempDownloadDir(self): self.assertEquals("/a/b.t.tmp", resolver._temp_download_dir("/a/b/", "t"))