def test_ist_set(self):
with tk.mktemp() as t1, tk.mktemp() as t2:
var1 = Variable(t1)
var2 = Variable(t2)
c1 = var1 + 5
c2 = c1 * 8
c3 = c2 - (var2 % 5)
c4 = c3.rformat('Hello {}')
for i in [var1, var2, c1, c2, c3, c4]:
self.assertEqual(i.is_set(), False)
var1.set(3)
for i in [var1, c1, c2]:
self.assertEqual(i.is_set(), True)
for i in [var2, c3, c4]:
self.assertEqual(i.is_set(), False)
var2.set(9)
for i in [var1, var2, c1, c2, c3, c4]:
self.assertEqual(i.is_set(), True)
self.check_only_get_eq(var1, 3)
self.check_only_get_eq(var2, 9)
self.check_only_get_eq(c1, 8)
self.check_only_get_eq(c2, 64)
self.check_only_get_eq(c3, 60)
self.check_only_get_eq(c4, 'Hello 60')
def pickle_and_check(path):
with tk.mktemp() as pickle_path:
with open(pickle_path, 'wb') as f:
pickle.dump(path, f)
with open(pickle_path, 'rb') as f:
path_unpickled = pickle.load(f)
self.assertEqual(path.__dict__, path_unpickled.__dict__)
def test_dir(self):
# create and automatically delete temp dir
with mktemp() as temp:
assert (len(glob.glob(temp)) == 0)
os.mkdir(temp)
assert (len(glob.glob(temp)) == 1)
assert (len(glob.glob(temp)) == 0)
def test_file(self):
# create and automatically delete temp file
with mktemp() as temp:
assert (len(glob.glob(temp)) == 0)
with open(temp, 'w'):
pass
assert (len(glob.glob(temp)) == 1)
assert (len(glob.glob(temp)) == 0)
def test_fallback(self):
with tk.mktemp() as t:
var = Variable(t)
self.assertEqual(var.is_set(), False)
fallback = var.fallback(0)
self.check_only_get_eq(fallback, 0)
self.check_only_get_eq(fallback + 5, 5)
var.set(3)
self.assertEqual(var.is_set(), True)
self.check_only_get_eq(fallback + 5, 8)
with tk.mktemp() as t:
var = Variable(t)
var_chain = ((var + 4) % 2) * 42
fallback = var_chain.rformat('{:05.1f}').fallback(0)
self.assertEqual(var.is_set(), False)
self.check_only_get_eq(fallback, 0)
var.set(3)
self.assertEqual(var.is_set(), True)
self.check_only_get_eq(fallback, '042.0')
def test_path_delay(self):
mjob = MockJob('test/me.1234')
path = Path('lm.gz', mjob)
self.check_only_get_eq(path, str(path))
self.check_only_get_eq(path + '.foo', str(path) + '.foo')
self.check_only_get_eq(path[:-3], str(path)[:-3])
self.check_only_get_eq(path[-2], str(path)[-2])
self.check_only_get_eq(path[:-3] + '.foo', str(path)[:-3] + '.foo')
with tk.mktemp() as t:
var = Variable(t)
var.set(3)
self.check_only_get_eq(var + 4, 7)
self.check_only_get_eq(4 + var, 7)
self.check_only_get_eq(var * 4, 12)
self.check_only_get_eq(4 * var, 12)
def test_path_available(self):
mjob = MockJob('test/me.1234')
path = Path('lm.gz', mjob)
self.assertEqual(path.available(), False)
mjob._sis_finished = lambda: True
mjob.path_available = lambda path: True
self.assertEqual(path.available(), True)
with tk.mktemp() as test_path:
path = Path(test_path)
self.assertEqual(path.available(), False)
with open(test_path, 'wb') as _:
pass
self.assertEqual(path.available(), True)
path = Path('lm.gz', mjob, available=path_available_false)
self.assertEqual(path.available(), False)
path = Path('lm.gz', mjob, available=path_available_true)
self.assertEqual(path.available(), True)
def remove_directories(dirs,
message,
move_postfix='.cleanup',
mode='remove',
force=False):
""" list all directories that will be deleted and add a security check """
if isinstance(dirs, str):
dirs = load_remove_list(dirs)
tmp = list(dirs)
tmp.sort(key=lambda x: str(x))
logging.info(message)
logging.info("Number of affected directories: %i" % len(dirs))
if len(dirs) == 0:
return
input_var = 'UNSET'
while input_var.lower() not in ('n', 'y', ''):
input_var = input("Calculate size of affected directories? (Y/n): ")
if input_var.lower() == 'n':
input_var = 'UNSET'
while input_var.lower() not in ('n', 'y', ''):
input_var = input("List affected directories? (Y/n): ")
if input_var.lower() != 'n':
logging.info("Affected directories:")
for i in tmp:
logging.info(i)
else:
# replace with 'with tempfile.namedtemporaryfile() as tmp_file:' when dropping support for older python verion
from sisyphus.toolkit import mktemp
with mktemp() as tmp_file_name:
with open(tmp_file_name, 'w') as tmp_file:
for directory in dirs:
tmp_file.write(directory + "\x00")
tmp_file.flush()
command = 'du -sch --files0-from=%s' % tmp_file_name
p = os.popen(command)
print(p.read())
p.close()
input_var = 'UNSET'
if force:
input_var = 'y'
while input_var.lower() not in ('n', 'y'):
message = 'Move directories?' if mode == 'move' else 'Delete directories?'
input_var = input("%s (y/n): " % message)
if input_var.lower() == 'y':
for num, k in enumerate(dirs, 1):
if mode == 'move':
logging.info('move: %s' % k)
# todo: k.{postfix} is may already used
shutil.move(k, k + '.' + move_postfix)
elif mode == 'remove':
logging.info('Delete: (%d/%d) %s' % (num, len(dirs), k))
if os.path.islink(k):
os.unlink(k)
else:
try:
shutil.rmtree(k)
except OSError as error:
print(error)
else:
assert False
else:
logging.error("Abort")