def add(self, job_specs: List[JobSpec]) -> None:
for job_spec in job_specs:
if io.exists(job_spec.output):
# TODO: we may need to do more than check for a single file
# TODO: we may want to check for timestamps here?
# TODO: we may want to support deleting files in events as a mechanism to triogger re-running? or not?
logging.info(
"Skipping enqueueing %s because its output file already exists!",
job_spec)
else:
serialized = job_spec.to_json()
deserialized = JobSpec.from_json(serialized)
deserialized.execute()
def main(argv):
del argv # Unused.
enqueuer = get_enqueuer()
if FLAGS.preflight_task:
logging.set_verbosity(logging.DEBUG)
src_path = FLAGS.preflight_task
parser = TaskParser(src_path)
task_spec = parser.to_spec()
logging.info("Task parsed successfully: %s", task_spec)
logging.info("Input dependencies: %s", task_spec.dependencies)
job_specs = task_spec.to_job_specs()
logging.info("Task %s could create %d jobs:", task_spec.name,
len(job_specs))
for job_spec in job_specs:
logging.info(str(job_spec))
enqueuer.add(job_specs)
elif FLAGS.execute_job:
src_path = FLAGS.execute_job
with open(src_path, 'r') as handle:
json = handle.read()
job_spec = JobSpec.from_json(json)
job_spec.execute()
elif FLAGS.watch:
event_handler = FileEventHandlerAdapterEventHandler(
root_dir=FLAGS.watch_path)
observer = Observer()
observer.schedule(event_handler, FLAGS.watch_path, recursive=True)
observer.start()
logging.info("Simulator ready, observing path '%s'", FLAGS.watch_path)
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
observer.stop()
observer.join()
else:
logging.fatal("No command given, see usage, etc TODO")
def test_deserialization_simple():
json = open('tests/fixtures/job_specs/simple.json').read()
new_spec = JobSpec.from_json(json)
assert new_spec is not None
# check that arrays get converted back to tuples:
assert new_spec.bindings == {"name": "Ludwig"}
def test_init_empty():
with raises(TypeError):
_ = JobSpec()
def simple_job_spec(): return JobSpec(**simple_job)
def noop_job_spec():
return JobSpec({"unity": 1}, "/data/noop", "/tasks/noop.py")
def to_job_spec(self, bindings: Bindings) -> "JobSpec":
str_bindings = stringify_bindings(bindings)
output_path = self.output_spec.with_replacements(str_bindings)
return JobSpec(bindings, output_path, self.src_path)
def handle_job_event(self, serialized_job_spec: str) -> None:
job_spec = JobSpec.from_json(serialized_job_spec)
job_spec.execute()