from prefect.tasks.shell import ShellTask from prefect.engine.executors import LocalDaskExecutor from prefect.environments import LocalEnvironment @task(max_retries=3, retry_delay=timedelta(seconds=0)) def extract_phizz(): return [ # schema output '''psql "dbname='db' user='******' password='******' host='postgres.rds.amazonaws.com'" -c "\COPY (select row_to_json(t) from public.information_schema.columns as t ) to '~/s3_bucket/extractload/schema.json';"''', ] getdata = ShellTask(name='shell task', helper_script='cd /home/ubuntu/prefect_scripts') with Flow("Schema Extract") as flow: phizz_data_extract = getdata.map(extract_phizz) flow.environment = LocalEnvironment( labels=[], executor=LocalDaskExecutor(scheduler="threads", num_workers=50), ) # to run locally use flow.run() #flow.run() # to register to the prefect server use flow.register # this assumes you have a project named ExtractLoad flow.register(project_name="ExtractLoad")
@task def render(object, name): return (f"{COMMAND} render set {object} " f"/uod/idr/metadata/idr0072-schormann-subcellref/" f"{name}/idr0072-{name}-render.yml") @task def list_children(name, ignore): with cli_login() as cli: conn = BlitzGateway(client_obj=cli.get_client()) screen = conn.getObject( 'Screen', attributes={'name': "idr0072-schormann-subcellref/" + name}) return [f"Plate:{x.id}" for x in screen.listChildren()] with Flow("idr0093") as flow: key = shell(command=f"{COMMAND} login demo@localhost") children = list_children(name, key) render_commands = render.map(children, name=unmapped(name)) rendered = shell.map(command=render_commands) if __name__ == "__main__": with raise_on_exception(): flow.executor = LocalDaskExecutor(scheduler="threads", num_workers=5) flow.run(name="screenA") flow.run(name="screenB")