def main():
backend_uri = None
tmp_path = None
try:
if example_utils.SQLALCHEMY_AVAILABLE:
tmp_path = tempfile.mktemp(prefix='tf-resume-example')
backend_uri = "sqlite:///%s" % (tmp_path)
else:
tmp_path = tempfile.mkdtemp(prefix='tf-resume-example')
backend_uri = 'file:///%s' % (tmp_path)
def run_example(name, add_env=None):
_exec([sys.executable, _path_to(name), backend_uri], add_env)
print('Run flow:')
run_example('run_flow.py')
print('\nRun flow, something happens:')
run_example('run_flow.py', {'BOOM': 'exit please'})
print('\nRun flow, something happens again:')
run_example('run_flow.py', {'BOOM': 'exit please'})
print('\nResuming all failed flows')
run_example('resume_all.py')
finally:
if tmp_path:
example_utils.rm_path(tmp_path)
def main():
backend_uri = None
tmp_path = None
try:
if example_utils.SQLALCHEMY_AVAILABLE:
tmp_path = tempfile.mktemp(prefix='tf-resume-example')
backend_uri = "sqlite:///%s" % (tmp_path)
else:
tmp_path = tempfile.mkdtemp(prefix='tf-resume-example')
backend_uri = 'file:///%s' % (tmp_path)
def run_example(name, add_env=None):
_exec([sys.executable, _path_to(name), backend_uri], add_env)
print('Run flow:')
run_example('run_flow.py')
print('\nRun flow, something happens:')
run_example('run_flow.py', {'BOOM': 'exit please'})
print('\nRun flow, something happens again:')
run_example('run_flow.py', {'BOOM': 'exit please'})
print('\nResuming all failed flows')
run_example('resume_all.py')
finally:
if tmp_path:
example_utils.rm_path(tmp_path)
def main():
backend_uri = None
tmp_path = None
try:
if example_utils.SQLALCHEMY_AVAILABLE:
tmp_path = tempfile.mktemp(prefix="tf-resume-example")
backend_uri = "sqlite:///%s" % (tmp_path)
else:
tmp_path = tempfile.mkdtemp(prefix="tf-resume-example")
backend_uri = "file:///%s" % (tmp_path)
def run_example(name, add_env=None):
_exec([sys.executable, _path_to(name), backend_uri], add_env)
print("Run flow:")
run_example("run_flow.py")
print("\nRun flow, something happens:")
run_example("run_flow.py", {"BOOM": "exit please"})
print("\nRun flow, something happens again:")
run_example("run_flow.py", {"BOOM": "exit please"})
print("\nResuming all failed flows")
run_example("resume_all.py")
finally:
if tmp_path:
example_utils.rm_path(tmp_path)
def main():
tmp_path = None
try:
tmp_path = tempfile.mkdtemp(prefix='worker-based-example-')
config = json.dumps({
'transport': 'filesystem',
'transport_options': {
'data_folder_in': tmp_path,
'data_folder_out': tmp_path
}
})
print('Run worker.')
worker_process, _ = run_test('worker.py', config)
print('Run flow.')
flow_process, flow_cmd = run_test('flow.py', config)
stdout, _ = flow_process.communicate()
rc = flow_process.returncode
if rc != 0:
raise RuntimeError("Could not run %s [%s]" % (flow_cmd, rc))
print(stdout.decode())
print('Flow finished.')
print('Stop worker.')
worker_process.terminate()
finally:
if tmp_path is not None:
example_utils.rm_path(tmp_path)
try:
# Create a set of workers to simulate actual remote workers.
print('Running %s workers.' % (worker_count))
for i in range(0, worker_count):
worker_conf['topic'] = 'worker-%s' % (i + 1)
worker_topics.append(worker_conf['topic'])
w = worker.Worker(**worker_conf)
runner = threading_utils.daemon_thread(w.run)
runner.start()
w.wait()
workers.append((runner, w.stop))
# Now use those workers to do something.
print('Executing some work.')
engine_options['topics'] = worker_topics
result = run(engine_options)
print('Execution finished.')
# This is done so that the test examples can work correctly
# even when the keys change order (which will happen in various
# python versions).
print("Result = %s" % json.dumps(result, sort_keys=True))
finally:
# And cleanup.
print('Stopping workers.')
while workers:
r, stopper = workers.pop()
stopper()
r.join()
if tmp_path:
example_utils.rm_path(tmp_path)
blowup = False
else:
blowup = True
with example_utils.get_backend(backend_uri) as backend:
# Now we can run.
engine_config = {
'backend': backend,
'engine_conf': 'serial',
'book': logbook.LogBook("my-test"),
}
# Make a flow that will blowup if the file doesn't exist previously, if it
# did exist, assume we won't blowup (and therefore this shows the undo
# and redo that a flow will go through).
flow = make_flow(blowup=blowup)
print_wrapped("Running")
try:
eng = engines.load(flow, **engine_config)
eng.run()
if not blowup:
example_utils.rm_path(persist_path)
except Exception:
# NOTE(harlowja): don't exit with non-zero status code, so that we can
# print the book contents, as well as avoiding exiting also makes the
# unit tests (which also runs these examples) pass.
traceback.print_exc(file=sys.stdout)
print_wrapped("Book contents")
print(p_utils.pformat(engine_config['book']))
backend_uri = "sqlite:///%s" % (persist_path)
else:
persist_path = os.path.join(tempfile.gettempdir(), "persisting")
backend_uri = "file:///%s" % (persist_path)
if os.path.exists(persist_path):
blowup = False
else:
blowup = True
with eu.get_backend(backend_uri) as backend:
# Make a flow that will blow up if the file didn't exist previously, if it
# did exist, assume we won't blow up (and therefore this shows the undo
# and redo that a flow will go through).
book = models.LogBook("my-test")
flow = make_flow(blowup=blowup)
eu.print_wrapped("Running")
try:
eng = engines.load(flow, engine='serial', backend=backend, book=book)
eng.run()
if not blowup:
eu.rm_path(persist_path)
except Exception:
# NOTE(harlowja): don't exit with non-zero status code, so that we can
# print the book contents, as well as avoiding exiting also makes the
# unit tests (which also runs these examples) pass.
traceback.print_exc(file=sys.stdout)
eu.print_wrapped("Book contents")
print(book.pformat())
blowup = False
else:
blowup = True
with example_utils.get_backend(backend_uri) as backend:
# Now we can run.
engine_config = {
'backend': backend,
'engine_conf': 'serial',
'book': logbook.LogBook("my-test"),
}
# Make a flow that will blowup if the file doesn't exist previously, if it
# did exist, assume we won't blowup (and therefore this shows the undo
# and redo that a flow will go through).
flow = make_flow(blowup=blowup)
print_wrapped("Running")
try:
eng = engines.load(flow, **engine_config)
eng.run()
if not blowup:
example_utils.rm_path(persist_path)
except Exception:
# NOTE(harlowja): don't exit with non-zero status code, so that we can
# print the book contents, as well as avoiding exiting also makes the
# unit tests (which also runs these examples) pass.
traceback.print_exc(file=sys.stdout)
print_wrapped("Book contents")
print(p_utils.pformat(engine_config['book']))
else:
persist_path = os.path.join(tempfile.gettempdir(), "persisting")
backend_uri = "file:///%s" % (persist_path)
if os.path.exists(persist_path):
blowup = False
else:
blowup = True
with eu.get_backend(backend_uri) as backend:
# Make a flow that will blow up if the file didn't exist previously, if it
# did exist, assume we won't blow up (and therefore this shows the undo
# and redo that a flow will go through).
book = logbook.LogBook("my-test")
flow = make_flow(blowup=blowup)
eu.print_wrapped("Running")
try:
eng = engines.load(flow, engine='serial',
backend=backend, book=book)
eng.run()
if not blowup:
eu.rm_path(persist_path)
except Exception:
# NOTE(harlowja): don't exit with non-zero status code, so that we can
# print the book contents, as well as avoiding exiting also makes the
# unit tests (which also runs these examples) pass.
traceback.print_exc(file=sys.stdout)
eu.print_wrapped("Book contents")
print(p_utils.pformat(book))
try:
# Create a set of workers to simulate actual remote workers.
print('Running %s workers.' % (worker_count))
for i in range(0, worker_count):
worker_conf['topic'] = 'worker-%s' % (i + 1)
worker_topics.append(worker_conf['topic'])
w = worker.Worker(**worker_conf)
runner = threading_utils.daemon_thread(w.run)
runner.start()
w.wait()
workers.append((runner, w.stop))
# Now use those workers to do something.
print('Executing some work.')
engine_options['topics'] = worker_topics
result = run(engine_options)
print('Execution finished.')
# This is done so that the test examples can work correctly
# even when the keys change order (which will happen in various
# python versions).
print("Result = %s" % json.dumps(result, sort_keys=True))
finally:
# And cleanup.
print('Stopping workers.')
while workers:
r, stopper = workers.pop()
stopper()
r.join()
if tmp_path:
example_utils.rm_path(tmp_path)
