def create_project_if_not_exists(client: prefect.Client, project_name: str) -> None: """Checks whether a project named "Monitorfish" already exists in Prefect Server. If not, the project is created. Args: client (prefect.Client): Prefect client instance Raises: ValueError: if more than 1 project with the name "Monitorfish" are found. """ r = client.graphql( 'query{project(where: {name: {_eq : "Monitorfish"}}){name}}') projects = r["data"]["project"] if len(projects) == 0: print("Monitorfish project does not exists, it will be created.") client.create_project(project_name) elif len(projects) == 1: print("Monitorfish project already exists. Skipping project creation.") else: raise ValueError( "Several projects with the name 'Monitorfish' were found.")
raise ("Failed to query " + api_endpoint) @task def get_weather(woeid: int): logger.info("Getting weather of {}".format(woeid)) api_endpoint = "https://www.metaweather.com/api/location/{}".format(woeid) response = requests.get(api_endpoint) if response.status_code == 200: weather_data = json.loads(response.text) logger.debug(weather_data) return weather_data else: raise ("Failed to query " + api_endpoint) with Flow("Get Paris' weather") as flow: woeid = get_woeid("Paris") weather_data = get_weather(woeid) try: client = Client() client.create_project(project_name="weather") except prefect.utilities.exceptions.ClientError as e: logger.info("Project already exists") flow.register(project_name="weather", labels=["development"]) # Optionally run the code now flow.run()
lambda flow: flow['project']['name']) } (root, projects, _) = next(walk(path.dirname(__file__))) for project_name in projects: if project_name in flows_by_project: for old_flow in flows_by_project[project_name].keys(): error = client.graphql(f'''mutation {{ archive_flow(input: {{ flow_id: "{old_flow}" }}) {{ error }} }}''').to_dict()['data']['archive_flow']['error'] print('Archiving existing flow %s.. %s' % (old_flow, error or 'OK')) files = next(walk(path.join(root, project_name)))[-1] for file in files: if not file.endswith('.py'): continue module = path.splitext(file)[0] flow = getattr( import_module('.%s.%s' % (project_name, module), __name__), 'flow') if callable(getattr(flow, 'register', None)): if (project_name not in flows_by_project) and file == files[0]: client.create_project(project_name) flow.register(project_name) flow.visualize(format='png', filename=path.join(root, project_name, module))
df = ph.read_clickhouse(query, connection=connection) return df @prefect.task(max_retries=5, retry_delay=timedelta(seconds=2)) def agregate(df): agg = df.groupby(['SalesDate', 'UserID'], as_index=False).sum() return agg # schedule to run every 12 hours schedule = IntervalSchedule( start_date=datetime.utcnow() + timedelta(seconds=1), # interval=timedelta(hours=12)) interval=timedelta(minutes=5), end_date=datetime.utcnow() + timedelta(minutes=10)) with prefect.Flow( name="SQL", schedule=schedule, # state_handlers=[handler], ) as flow: dataframes = download(connection) fin = agregate(dataframes) # client = Client() client.create_project(project_name='SQL') flow.register(project_name='SQL') # flow.run()