def wait_for_input_files(self):
''' wait for all the input files to be available and update input_files_local '''
for fl in self.input_files:
flist = []
for f in fl:
if f in self._files_seen:
raise InvalidInputFileError({
"error":
"Filename conflict: {} already exists in {}".format(
f, self._files_seen),
"step":
self.name
})
else:
self._files_seen.add(f)
local_file = os.path.join(self.output_dir_local, f)
while True:
if os.path.exists(local_file) and os.path.exists(
self.done_file(local_file)):
flist.append(local_file)
break
else:
if self.should_terminate:
# If the step is not supposed to be run any more.
raise RuntimeError("Step %s being terminated" %
self.name)
time.sleep(5)
self.input_files_local.append(flist)
def wait_until_finished(self):
self.exec_thread.join()
if self.status == StepStatus.INVALID_INPUT:
raise InvalidInputFileError({
"error": self.input_file_error.name,
"step": self.name
})
if self.status < StepStatus.FINISHED:
raise RuntimeError("step %s run failed" % self.name)