from prefect.tasks.shell import ShellTask
from prefect.engine.executors import LocalDaskExecutor
from prefect.environments import LocalEnvironment
@task(max_retries=3, retry_delay=timedelta(seconds=0))
def extract_phizz():
return [
# schema output
'''psql "dbname='db' user='******' password='******' host='postgres.rds.amazonaws.com'" -c "\COPY (select row_to_json(t) from public.information_schema.columns as t ) to '~/s3_bucket/extractload/schema.json';"''',
]
getdata = ShellTask(name='shell task',
helper_script='cd /home/ubuntu/prefect_scripts')
with Flow("Schema Extract") as flow:
phizz_data_extract = getdata.map(extract_phizz)
flow.environment = LocalEnvironment(
labels=[],
executor=LocalDaskExecutor(scheduler="threads", num_workers=50),
)
# to run locally use flow.run()
#flow.run()
# to register to the prefect server use flow.register
# this assumes you have a project named ExtractLoad
flow.register(project_name="ExtractLoad")
@task
def render(object, name):
return (f"{COMMAND} render set {object} "
f"/uod/idr/metadata/idr0072-schormann-subcellref/"
f"{name}/idr0072-{name}-render.yml")
@task
def list_children(name, ignore):
with cli_login() as cli:
conn = BlitzGateway(client_obj=cli.get_client())
screen = conn.getObject(
'Screen',
attributes={'name': "idr0072-schormann-subcellref/" + name})
return [f"Plate:{x.id}" for x in screen.listChildren()]
with Flow("idr0093") as flow:
key = shell(command=f"{COMMAND} login demo@localhost")
children = list_children(name, key)
render_commands = render.map(children, name=unmapped(name))
rendered = shell.map(command=render_commands)
if __name__ == "__main__":
with raise_on_exception():
flow.executor = LocalDaskExecutor(scheduler="threads", num_workers=5)
flow.run(name="screenA")
flow.run(name="screenB")