def add(self, job_specs: List[JobSpec]) -> None: for job_spec in job_specs: if io.exists(job_spec.output): # TODO: we may need to do more than check for a single file # TODO: we may want to check for timestamps here? # TODO: we may want to support deleting files in events as a mechanism to triogger re-running? or not? logging.info( "Skipping enqueueing %s because its output file already exists!", job_spec) else: serialized = job_spec.to_json() deserialized = JobSpec.from_json(serialized) deserialized.execute()
def main(argv): del argv # Unused. enqueuer = get_enqueuer() if FLAGS.preflight_task: logging.set_verbosity(logging.DEBUG) src_path = FLAGS.preflight_task parser = TaskParser(src_path) task_spec = parser.to_spec() logging.info("Task parsed successfully: %s", task_spec) logging.info("Input dependencies: %s", task_spec.dependencies) job_specs = task_spec.to_job_specs() logging.info("Task %s could create %d jobs:", task_spec.name, len(job_specs)) for job_spec in job_specs: logging.info(str(job_spec)) enqueuer.add(job_specs) elif FLAGS.execute_job: src_path = FLAGS.execute_job with open(src_path, 'r') as handle: json = handle.read() job_spec = JobSpec.from_json(json) job_spec.execute() elif FLAGS.watch: event_handler = FileEventHandlerAdapterEventHandler( root_dir=FLAGS.watch_path) observer = Observer() observer.schedule(event_handler, FLAGS.watch_path, recursive=True) observer.start() logging.info("Simulator ready, observing path '%s'", FLAGS.watch_path) try: while True: time.sleep(1) except KeyboardInterrupt: observer.stop() observer.join() else: logging.fatal("No command given, see usage, etc TODO")
def test_deserialization_simple(): json = open('tests/fixtures/job_specs/simple.json').read() new_spec = JobSpec.from_json(json) assert new_spec is not None # check that arrays get converted back to tuples: assert new_spec.bindings == {"name": "Ludwig"}
def handle_job_event(self, serialized_job_spec: str) -> None: job_spec = JobSpec.from_json(serialized_job_spec) job_spec.execute()