def update_mapping(self, flush=False):
last_mapped_input = None
if len(self.io_mapping) >= 1:
last_mapped_input = self.io_mapping[-1][0][-1]
num = 0
chunk = []
output_idx = len(self.io_mapping)
start_idx = 0 if not last_mapped_input else (
self.inputs.index(last_mapped_input) + 1)
for inp in self.inputs[start_idx:]:
if num >= self.files_per_output:
# push in new chunk
self.io_mapping.append(
[chunk[:], [File(self.output_pattern.format(output_idx))]])
# reset current chunk variables
output_idx += 1
num = 0
chunk = []
chunk.append(inp)
num += 1
# push remaining partial chunk if flush is True
if (len(chunk) == self.files_per_output) or (flush and len(chunk) > 0):
self.io_mapping.append(
[chunk[:], [File(self.output_pattern.format(output_idx))]])
def test_name_manipulations(self):
f1 = File("/tmp/does_not_exist_1.root", fake=True)
self.assertEqual(f1.get_extension(), "root")
self.assertEqual(f1.get_basepath(), "/tmp")
self.assertEqual(f1.get_basename(), "does_not_exist_1.root")
self.assertEqual(f1.get_basename_noext(), "does_not_exist_1")
self.assertEqual(f1.get_index(), 1)
f2 = File("test.root", fake=True)
self.assertEqual(f2.get_basepath(), ".")
def test_dependencies(self):
t1 = Task(a=1)
t2 = Task(a=2)
t3 = Task(a=3, requirements=[t1, t2])
self.assertEqual(t3.get_requirements(), [t1, t2])
f1 = File("outputblah1.txt")
f2 = File("outputblah2.txt")
t1.get_outputs = lambda: [f1]
t2.get_outputs = lambda: [f2]
self.assertEqual(t1.complete(), False)
self.assertEqual(t2.complete(), False)
self.assertEqual(t3.requirements_satisfied(), False)
f1.set_fake()
self.assertEqual(t1.complete(), True)
self.assertEqual(t2.complete(), False)
self.assertEqual(t3.requirements_satisfied(), False)
f2.set_fake()
self.assertEqual(t1.complete(), True)
self.assertEqual(t2.complete(), True)
self.assertEqual(t3.requirements_satisfied(), True)
def update_mapping(self):
if self.io_mapping: return
self.io_mapping = [[
map(File, self.input_filenames), [
File(self.output_filename),
]
]]
def test_set_name(self):
name = "/tmp/does_not_exist_1.root"
new_name = "/tmp/does_not_exist_2.root"
f = File(name)
self.assertEqual(f.get_name(), name)
f.set_name(new_name)
self.assertEqual(f.get_name(), new_name)
def test_recheck_real(self):
# make a test file
basedir = "/tmp/{0}/metis/file_test/".format(os.getenv("USER"))
fname = "{0}/test.txt".format(basedir)
Utils.do_cmd("mkdir -p {0}".format(basedir))
Utils.do_cmd("touch {0}".format(fname))
f = File(fname)
# it exists
self.assertEqual(f.exists(), True)
# delete it
Utils.do_cmd("rm {0}".format(fname))
# it still exists due to caching (to avoid unnecessary `ls`)
self.assertEqual(f.exists(), True)
# force recheck/recache
f.recheck()
# now it doesn't exist
self.assertEqual(f.exists(), False)
def test_workflow(self):
step0 = []
for i in range(24):
step0.append(File(name="step0_{}.root".format(i)))
t1 = CombinerTask(
files_per_output=2,
output_pattern="step1_{}.root",
create_inputs=True,
)
t1.add_inputs(step0, flush=True)
self.assertEqual(len(t1.get_outputs()), 12)
t2 = CombinerTask(
files_per_output=3,
output_pattern="step2_{}.root",
)
t2.add_inputs(t1.get_outputs(), flush=True)
self.assertEqual(len(t2.get_outputs()), 4)
# print t2.get_outputs()
t3 = CombinerTask(
files_per_output=4,
output_pattern="step2_{}.root",
)
t3.add_inputs(t2.get_outputs(), flush=True)
self.assertEqual(len(t3.get_outputs()), 1)
t1.run()
t2.run()
t3.run()
self.assertEqual(t3.get_outputs()[0].exists(), True)
def test_condor_handler(self):
epsilon_hours = 0.1
remove_running_x_hours = 36.
remove_held_x_hours = 3.
params = {
"out": File("blah"),
"fake": True,
"remove_running_x_hours": remove_running_x_hours,
"remove_held_x_hours": remove_held_x_hours,
}
job_dict = {
"ClusterId": 123,
"ProcId": 0,
"JobStatus": "R",
"EnteredCurrentStatus": time.time()
}
self.assertEqual(
self.dummy.handle_condor_job(this_job_dict=job_dict, **params),
"RUNNING")
job_dict = {
"ClusterId":
123,
"ProcId":
0,
"JobStatus":
"R",
"EnteredCurrentStatus":
time.time() - (remove_running_x_hours - epsilon_hours) * 3600
}
self.assertEqual(
self.dummy.handle_condor_job(this_job_dict=job_dict, **params),
"RUNNING")
job_dict = {
"ClusterId":
123,
"ProcId":
0,
"JobStatus":
"R",
"EnteredCurrentStatus":
time.time() - (remove_running_x_hours + epsilon_hours) * 3600
}
self.assertEqual(
self.dummy.handle_condor_job(this_job_dict=job_dict, **params),
"LONG_RUNNING_REMOVED")
job_dict = {
"ClusterId": 123,
"ProcId": 0,
"JobStatus": "I",
"EnteredCurrentStatus": time.time()
}
self.assertEqual(
self.dummy.handle_condor_job(this_job_dict=job_dict, **params),
"IDLE")
job_dict = {
"ClusterId":
123,
"ProcId":
0,
"JobStatus":
"H",
"EnteredCurrentStatus":
time.time() - (remove_held_x_hours - epsilon_hours) * 3600
}
self.assertEqual(
self.dummy.handle_condor_job(this_job_dict=job_dict, **params),
"HELD")
job_dict = {
"ClusterId":
123,
"ProcId":
0,
"JobStatus":
"H",
"EnteredCurrentStatus":
time.time() - (remove_held_x_hours + epsilon_hours) * 3600
}
self.assertEqual(
self.dummy.handle_condor_job(this_job_dict=job_dict, **params),
"HELD_AND_REMOVED")
def test_file_exists(self):
f = File(__file__)
self.assertEqual(f.exists(), True)
def test_status(self):
f = File("does_not_exist.root", status=Constants.VALID)
self.assertEqual(f.get_status(), Constants.VALID)
f.set_status(Constants.INVALID)
self.assertEqual(f.get_status(), Constants.INVALID)
def test_file_notequal(self):
f1 = File("does_not_exist1.root")
f2 = File("does_not_exist2.root")
self.assertNotEqual(f1, f2)
def test_file_equals(self):
f1 = File("does_not_exist0.root")
f2 = File("does_not_exist0.root")
self.assertEqual(f1, f2.get_name())
self.assertEqual(f1, f2)
def test_recheck_fake(self):
f = File("does_not_exist.root", fake=True)
self.assertEqual(f.exists(), True)
f.recheck()
self.assertEqual(f.exists(), True)
def test_unset_fake(self):
f = File("does_not_exist.root", fake=True)
self.assertEqual(f.exists(), True)
f.unset_fake()
self.assertEqual(f.exists(), False)
def test_not_exists(self):
f = File("does_not_exist.root")
self.assertEqual(f.exists(), False)