def run(
self,
key: str,
aws_credentials_secret: str = "AWS_CREDENTIALS",
bucket: str = None,
):
"""
Task run method.
Args:
- key (str): the name of the Key within this bucket to retrieve
- aws_credentials_secret (str, optional): the name of the Prefect Secret
which stores your AWS credentials; this Secret must be a JSON string
with two keys: `ACCESS_KEY` and `SECRET_ACCESS_KEY`
- bucket (str, optional): the name of the S3 Bucket to download from
Returns:
- str: the contents of this Key / Bucket, as a string
"""
if bucket is None:
raise ValueError("A bucket name must be provided.")
## get AWS credentials
aws_credentials = Secret(aws_credentials_secret).get()
aws_access_key = aws_credentials["ACCESS_KEY"]
aws_secret_access_key = aws_credentials["SECRET_ACCESS_KEY"]
s3_client = boto3.client(
"s3",
aws_access_key_id=aws_access_key,
aws_secret_access_key=aws_secret_access_key,
)
stream = io.BytesIO()
## download
s3_client.download_fileobj(Bucket=bucket, Key=key, Fileobj=stream)
## prepare data and return
stream.seek(0)
output = stream.read()
return output.decode()
def initialize_service(self) -> None:
"""
Initialize a Blob service.
"""
import azure.storage.blob
azure_credentials = Secret(self.azure_credentials_secret).get()
if isinstance(azure_credentials, str):
azure_credentials = json.loads(azure_credentials)
az_account_name = azure_credentials["ACCOUNT_NAME"]
az_account_key = azure_credentials["ACCOUNT_KEY"]
az_sas_token = azure_credentials["SAS_TOKEN"]
if az_sas_token is None:
blob_service = azure.storage.blob.BlockBlobService(
account_name=az_account_name, account_key=az_account_key)
else:
blob_service = azure.storage.blob.BlockBlobService(
account_name=az_account_name, sas_token=az_sas_token)
self.service = blob_service
def run(self, message: Union[str, dict] = None) -> None:
"""
Run method which sends a Slack message.
Args:
- message (Union[str,dict], optional): the message to send as either a dictionary or a plain string;
if not provided here, will use the value provided at initialization
Returns:
- None
"""
# 'import requests' is expensive time-wise, we should do this just-in-time to keep
# the 'import prefect' time low
import requests
webhook_url = cast(str, Secret(self.webhook_secret).get())
r = requests.post(
webhook_url,
json=message if isinstance(message, dict) else {"text": message},
)
r.raise_for_status()
def run(self, msg: str = None, access_token: str = None) -> None:
"""
Run method for this Task. Invoked by calling this Task after initialization within a
Flow context, or by using `Task.bind`.
Args:
- msg (str): The message you want sent to your phone; defaults to the one provided
at initialization
- access_token (str): a Pushbullet access token, provided with a Prefect secret.
Defaults to the "PUSHBULLET_TOKEN" secret
"""
if access_token is None:
access_token = Secret("PUSHBULLET_TOKEN").get()
pb = Pushbullet(access_token)
if msg is None:
raise ValueError("A message must be provided")
# send the request
pb.push_note("Flow Notification", msg)
def run(self, aws_credentials_secret: str = "AWS_CREDENTIALS"):
"""
Task fun method. Lists all Lambda functions.
Args:
- aws_credentials_secret (str, optional): the name of the Prefect Secret
that stores your AWS credentials; this Secret must be a JSON string
with two keys: `ACCESS_KEY` and `SECRET_ACCESS_KEY`
Returns:
- dict : a list of Lambda functions from AWS ListFunctions endpoint
"""
## get AWS credentials
aws_credentials = Secret(aws_credentials_secret).get()
aws_access_key = aws_credentials["ACCESS_KEY"]
aws_secret_access_key = aws_credentials["SECRET_ACCESS_KEY"]
lambda_client = boto3.client(
"lambda",
aws_access_key_id=aws_access_key,
aws_secret_access_key=aws_secret_access_key,
)
## list functions, optionally passing in marker if not None
if self.marker:
response = lambda_client.list_functions(
MasterRegion=self.master_region,
FunctionVersion=self.function_version,
Marker=self.marker,
MaxItems=self.max_items,
)
return response
response = lambda_client.list_functions(
MasterRegion=self.master_region,
FunctionVersion=self.function_version,
MaxItems=self.max_items,
)
return response
def run(
self,
namespace: str = "default",
kube_kwargs: dict = None,
kubernetes_api_key_secret: str = "KUBERNETES_API_KEY",
) -> None:
"""
Task run method.
Args:
- namespace (str, optional): The Kubernetes namespace to list pods from,
defaults to the `default` namespace
- kube_kwargs (dict, optional): Optional extra keyword arguments to pass to the
Kubernetes API (e.g. `{"field_selector": "...", "label_selector": "..."}`)
- kubernetes_api_key_secret (str, optional): the name of the Prefect Secret
which stored your Kubernetes API Key; this Secret must be a string and in
BearerToken format
Returns:
- V1PodList: a Kubernetes V1PodList of the pods which are found
"""
kubernetes_api_key = None
if kubernetes_api_key_secret:
kubernetes_api_key = Secret(kubernetes_api_key_secret).get()
if kubernetes_api_key:
configuration = client.Configuration()
configuration.api_key["authorization"] = kubernetes_api_key
api_client = client.CoreV1Api(client.ApiClient(configuration))
else:
try:
config.load_incluster_config()
except ConfigException:
config.load_kube_config()
api_client = client.CoreV1Api()
kube_kwargs = {**self.kube_kwargs, **(kube_kwargs or {})}
return api_client.list_namespaced_pod(namespace=namespace, **kube_kwargs)
def _get_blob(
self,
bucket: str,
blob: str,
encryption_key: str = None,
encryption_key_secret: str = None,
):
"Retrieves blob based on user settings."
if blob is None:
blob = "prefect-" + context.get("task_run_id",
"no-id-" + str(uuid.uuid4()))
## pull encryption_key if requested
if encryption_key_secret is not None:
warnings.warn(
"The `encryption_key_secret` argument is deprecated. Use a `Secret` task "
"to pass the credentials value at runtime instead.",
UserWarning,
)
encryption_key = Secret(encryption_key_secret).get()
return bucket.blob(blob, encryption_key=encryption_key)
def run(
self,
data: str,
blob_name: str = None,
azure_credentials_secret: str = "AZ_CONNECTION_STRING",
container: str = None,
) -> str:
"""
Task run method.
Args:
- data (str): the data payload to upload
- blob_name (str, optional): the name to upload the data under; if not
provided, a random `uuid` will be created
- azure_credentials_secret (str, optional): the name of the Prefect Secret
that stores your Azure credentials; this Secret must be an Azure connection string
- container (str, optional): the name of the Blob Storage container to upload to
Returns:
- str: the name of the blob the data payload was uploaded to
"""
if container is None:
raise ValueError("A container name must be provided.")
# get Azure credentials
azure_credentials = Secret(azure_credentials_secret).get()
blob_service = azure.storage.blob.BlockBlobService(conn_str=azure_credentials)
# create key if not provided
if blob_name is None:
blob_name = str(uuid.uuid4())
client = blob_service.get_blob_client(container=container, blob=blob_name)
client.upload_blob(data)
return blob_name
def _get_client(self,
project: str,
credentials: dict,
credentials_secret: str = None):
"""
Creates and returns a GCS Client instance
"""
if credentials_secret is not None:
warnings.warn(
"The `credentials_secret` argument is deprecated. Use a `Secret` task "
"to pass the credentials value at runtime instead.",
UserWarning,
)
creds = Secret(credentials_secret).get()
credentials = Credentials.from_service_account_info(creds)
project = project or credentials.project_id
if credentials is not None:
project = project or credentials.get("project")
client = storage.Client(project=project, credentials=credentials)
else:
client = storage.Client(project=project)
return client
def run(self, path: str = None, access_token_secret: str = None) -> bytes:
"""
Run method for this Task. Invoked by _calling_ this Task within a Flow context, after initialization.
Args:
- path (str, optional): the path to the file to download
- access_token_secret (str, optional): the name of the Prefect Secret containing a
Dropbox access token; defaults to `"DROPBOX_ACCESS_TOKEN"`
Raises:
- ValueError: if the `path` is `None`
Returns:
- bytes: the file contents, as bytes
"""
## check for any argument inconsistencies
if path is None:
raise ValueError("No path provided.")
access_token = Secret(access_token_secret).get()
dbx = dropbox.Dropbox(access_token)
response = dbx.files_download(path)[1]
return response.content
def initialize_client(self) -> None:
"""
Initializes an S3 Client.
"""
import boto3
aws_access_key = None
aws_secret_access_key = None
if self.aws_credentials_secret:
aws_credentials = Secret(self.aws_credentials_secret).get()
if isinstance(aws_credentials, str):
aws_credentials = json.loads(aws_credentials)
aws_access_key = aws_credentials["ACCESS_KEY"]
aws_secret_access_key = aws_credentials["SECRET_ACCESS_KEY"]
s3_client = boto3.client(
"s3",
aws_access_key_id=aws_access_key,
aws_secret_access_key=aws_secret_access_key,
)
self.client = s3_client
def run(
self,
path: str = None,
access_token: str = None,
access_token_secret: str = None,
) -> bytes:
"""
Run method for this Task. Invoked by _calling_ this Task within a Flow context, after
initialization.
Args:
- path (str, optional): the path to the file to download
- access_token (str): a Dropbox access token, provided with a Prefect secret.
- access_token_secret (str, optional, DEPRECATED): the name of the Prefect Secret
containing a Dropbox access token
Raises:
- ValueError: if the `path` is `None`
Returns:
- bytes: the file contents, as bytes
"""
# check for any argument inconsistencies
if path is None:
raise ValueError("No path provided.")
if access_token_secret is not None:
warnings.warn(
"The `access_token_secret` argument is deprecated. Use a `Secret` task "
"to pass the credentials value at runtime instead.",
UserWarning,
stacklevel=2,
)
access_token = Secret(access_token_secret).get()
dbx = dropbox.Dropbox(access_token)
response = dbx.files_download(path)[1]
return response.content
def run(
self,
blob_name: str,
azure_credentials_secret: str = "AZ_CREDENTIALS",
container: str = None,
) -> str:
"""
Task run method.
Args:
- blob_name (str): the name of the blob within this container to retrieve
- azure_credentials_secret (str, optional): the name of the Prefect Secret
that stores your Azure credentials; this Secret must be a JSON string
with two keys: `ACCOUNT_NAME` and `ACCOUNT_KEY`
- container (str, optional): the name of the Blob Storage container to download from
Returns:
- str: the contents of this blob_name / container, as a string
"""
if container is None:
raise ValueError("A container name must be provided.")
# get Azure credentials
azure_credentials = Secret(azure_credentials_secret).get()
az_account_name = azure_credentials["ACCOUNT_NAME"]
az_account_key = azure_credentials["ACCOUNT_KEY"]
blob_service = azure.storage.blob.BlockBlobService(
account_name=az_account_name, account_key=az_account_key)
blob_result = blob_service.get_blob_to_text(container_name=container,
blob_name=blob_name)
content_string = blob_result.content
return content_string
def run(
self,
data: dict,
base_key: str = None,
table_name: str = None,
credentials_secret: str = None,
) -> dict:
"""
Inserts data into an Airtable table
Args:
- data (dict): the data to insert. This should be formatted as a dictionary mapping
each column name to a value.
- base_key (str): the Airtable base key
- table_name (str): the table name
- credentials_secret (str): the name of a secret that contains an Airtable API key.
Defaults to "AIRTABLE_API_KEY"
Returns:
- a dictionary containing information about the successful insert
"""
api_key = Secret(credentials_secret).get()
table = airtable.Airtable(base_key, table_name, api_key)
return table.insert(data)
def run(
self,
repo: str = None,
from_path: str = None,
to_path: str = None,
access_token_secret: str = None,
branch: str = None,
):
"""Task run method.
Args:
repo (str, optional): The repository in the format `org/repo`. Defaults to None.
from_path (str, optional): The path to the file. Defaults to None.
to_path (str, optional): The destination path. Defaults to None.
access_token_secret (str, optional): The Prefect secret containing GitHub token. Defaults to "github_token".
branch (str, optional): The GitHub branch to use. Defaults to "main".
"""
git_token = Secret(access_token_secret).get()
file_name = from_path.split("/")[-1]
to_path = to_path or file_name
g = Github(git_token)
repo_obj = g.get_repo(repo)
try:
content_encoded = repo_obj.get_contents(
urllib.parse.quote(from_path), ref=branch).content
except UnknownObjectException as e:
full_path = os.path.join(repo, from_path)
raise ValueError(
f"The specified file does not exist under {full_path}.")
content = base64.b64decode(content_encoded)
if os.path.dirname(to_path):
os.makedirs(os.path.dirname(to_path), exist_ok=True)
with open(to_path, "w") as f:
f.write(content.decode())
import pandas as pd
import os
import psycopg2
import prefect
from prefect import Flow, task
from prefect.client import Secret
from sqlalchemy import create_engine
from google.cloud import storage
db_port = Secret("db_port")
db_user = Secret("db_user")
db_pass = Secret("db_pass")
db_host = Secret("db_host")
db_db = Secret("db_db")
storage_client = storage.Client()
@task
def extract(extract_name):
extract_path = f'gs://small-world-analytics-filestore/quantified-self/{extract_name}.csv'
df = pd.read_csv(extract_path)
return df
@task
def transform(df):
df.columns = df.columns.str.strip('() ')
return df
def run(
self,
username: str = None,
access_token: str = None,
server_url: str = None,
project_name: str = None,
assignee: str = "-1",
issue_type: str = None,
summary: str = None,
description: str = None,
) -> None:
"""
Run method for this Task. Invoked by calling this Task after initialization within a
Flow context, or by using `Task.bind`.
Args:
- username(str): the jira username, provided with a Prefect secret (defaults to
JIRAUSER in JIRASECRETS)
- access_token (str): a Jira access token, provided with a Prefect secret (defaults
to JIRATOKEN in JIRASECRETS)
- server_url (str): the URL of your atlassian account e.g.
"https://test.atlassian.net". Can also be set as a Prefect Secret. Defaults to
the one provided at initialization
- project_name(str): the key for your jira project; defaults to the one provided
at initialization
- assignee (str, optional): the atlassian accountId of the person you want to
assign the ticket to; defaults to "automatic" if this is not set; defaults to
the one provided at initialization
- issue_type (str, optional): the type of issue you want to create; defaults to
'Task'
- summary (str, optional): summary or title for your issue; defaults to the one
provided at initialization
- description (str, optional): description or additional information for the issue;
defaults to the one provided at initialization
Raises:
- ValueError: if a `project_name` or 'summary' are not provided
Returns:
- None
"""
try:
from jira import JIRA
except ImportError as exc:
raise ImportError(
'Using Jira tasks requires Prefect to be installed with the "jira" extra.'
) from exc
jira_credentials = cast(dict, Secret("JIRASECRETS").get())
if username is None:
username = jira_credentials["JIRAUSER"]
if access_token is None:
access_token = jira_credentials["JIRATOKEN"]
if server_url is None:
server_url = jira_credentials["JIRASERVER"]
if issue_type is None:
issue_type = "Task"
if project_name is None:
raise ValueError("A project name must be provided")
if summary is None:
raise ValueError("A summary must be provided")
jira = JIRA(basic_auth=(username, access_token),
options={"server": server_url})
options = {
"project": project_name,
"assignee": {
"accountId": assignee
},
"issuetype": {
"name": issue_type
},
"summary": summary,
"description": description,
}
created = jira.create_issue(options)
if not created:
raise ValueError("Creating Jira Issue failed")
def run(
self,
from_email: str = "*****@*****.**",
to_emails: Union[str, Tuple[str, str], List[str], List[Tuple[str, str]]] = None,
subject: str = None,
html_content: str = None,
category: Union[str, List[str]] = None,
attachment_file_path: Union[str, Path] = None,
sendgrid_secret: str = None,
) -> Response:
"""
Run message which sends an email via SendGrid.
Args:
- from_email (str): The email address of the sender;
defaults to the one provided at initialization
- to_emails (Union[str, Tuple[str, str], List[str], List[Tuple[str, str]]]):
The email address of the recipient(s); defaults to the one provided at initialization.
Refer to [SendGrid-Python](https://github.com/sendgrid/sendgrid-python) for specifics.
- subject (str, optional): The subject of the email;
defaults to the one provided at initialization
- html_content (str): The html body of the email;
defaults to the one provided at initialization
- category (Union[str, List[str]], optional): The category/categories to use for the email;
defaults to those provided at initialization
- attachment_file_path (Union[str, Path], optional): The file path of the email attachment;
defaults to the one provided at initialization
- sendgrid_secret (str, optional): the name of the Prefect Secret which stores your
SendGrid API key; defaults to `"SENDGRID_API_KEY"`; if not provided here,
will use the value provided at initialization
Returns:
- python_http_client.client.Response:
A [Python-HTTP-Client](https://github.com/sendgrid/python-http-client) object
indicating the status of the response
"""
# Based on the SendGrid example use-case code here:
# https://github.com/sendgrid/sendgrid-python/blob/aa39f715a061f0de993811faea0adb8223657d01/use_cases/attachment.md
sendgrid_api_key = Secret(sendgrid_secret).get()
import base64
import mimetypes
from sendgrid.helpers.mail import (
Attachment,
Category,
Disposition,
FileContent,
FileName,
FileType,
Mail,
)
from sendgrid import SendGridAPIClient
message = Mail(
from_email=from_email,
to_emails=to_emails,
subject=subject,
html_content=html_content,
)
if category:
if not isinstance(category, list):
category = [category]
message.category = [Category(str(c)) for c in category]
if attachment_file_path:
with open(attachment_file_path, "rb") as f:
data = f.read()
f.close()
encoded = base64.b64encode(data).decode()
guessed_type, content_encoding = mimetypes.guess_type(
attachment_file_path, strict=True
)
attachment = Attachment()
attachment.file_content = FileContent(encoded)
attachment.file_type = FileType(guessed_type)
attachment.file_name = FileName(Path(attachment_file_path).name)
attachment.disposition = Disposition("attachment")
message.attachment = attachment
sendgrid_client = SendGridAPIClient(sendgrid_api_key)
response = sendgrid_client.send(message)
return response
def test_secret_raises_if_doesnt_exist():
secret = Secret(name="test")
with set_temporary_config({"cloud.use_local_secrets": True}):
with pytest.raises(ValueError) as exc:
secret.get()
assert "not found" in str(exc.value)
def test_secrets_dont_raise_just_because_flow_key_is_populated():
secret = Secret(name="test")
with set_temporary_config({"cloud.use_local_secrets": True}):
with prefect.context(secrets=dict(test=42), flow="not None"):
assert secret.get() == 42
def test_secret_raises_if_doesnt_exist():
secret = Secret(name="test")
with set_temporary_config({"cloud.use_local_secrets": True}):
with pytest.raises(ValueError, match="not found"):
secret.get()
def run(
self,
subject: str = None,
msg: str = None,
email_to: str = None,
email_from: str = None,
smtp_server: str = None,
smtp_port: int = None,
smtp_type: str = None,
) -> None:
"""
Run method which sends an email.
Args:
- subject (str, optional): the subject of the email; defaults to the one provided
at initialization
- msg (str, optional): the contents of the email; defaults to the one provided
at initialization
- email_to (str, optional): the destination email address to send the message to;
defaults to the one provided at initialization
- email_from (str, optional): the email address to send from; defaults to the one
provided at initialization
- smtp_server (str, optional): the hostname of the SMTP server; defaults to the one
provided at initialization
- smtp_port (int, optional): the port number of the SMTP server; defaults to the one
provided at initialization
- smtp_type (str, optional): either SSL or STARTTLS; defaults to the one provided
at initialization
Returns:
- None
"""
username = cast(str, Secret("EMAIL_USERNAME").get())
password = cast(str, Secret("EMAIL_PASSWORD").get())
email_to = cast(str, email_to)
contents = MIMEMultipart("alternative")
contents.attach(MIMEText(cast(str, msg), "html"))
contents["Subject"] = Header(subject, "UTF-8")
contents["From"] = email_from
contents["To"] = email_to
message = contents.as_string()
context = ssl.create_default_context()
if smtp_type == "SSL":
server = smtplib.SMTP_SSL(smtp_server, smtp_port, context=context)
elif smtp_type == "STARTTLS":
server = smtplib.SMTP(smtp_server, smtp_port)
server.starttls(context=context)
else:
raise ValueError(
f"{smtp_type} is an unsupported value for smtp_type.")
server.login(username, password)
try:
server.sendmail(email_from, email_to, message)
finally:
server.quit()
def test_create_secret():
secret = Secret(name="test")
assert secret
def run(
self,
user: str = "",
password: str = "",
dsn: str = "",
query: str = None,
query_params: dict = None,
autocommit: bool = False,
commit: bool = True,
**kwargs,
):
"""
Task run method. Executes a query against Exasol database.
Args:
- user (str, optional): user name used to authenticate. This overrides the
initial `user_secret` parameter (if set)
- password (str, optional): password used to authenticate. This overrides the
initial `user_secret` parameter (if set)
- dsn (str, optional): dsn string of the database (server:port)
This overrides the initial `user_secret` parameter (if set)
- query (str, optional): query to execute against database
- query_params (dict, optional): Values for SQL query placeholders
- autocommit (bool, optional): turn autocommit on or off (default: False)
- commit (bool, optional): set to True to commit transaction, defaults to True
(only necessary if autocommit = False)
- **kwargs (dict, optional): additional connection parameter (connection_timeout...)
Returns:
- Nothing
Raises:
- ValueError: if dsn string is not provided
- ValueError: if query parameter is None or a blank string
- Exa*Error: multiple exceptions raised from the underlying pyexasol package
(e.g. ExaQueryError, ExaAuthError..)
"""
if not dsn and not self.dsn_secret:
raise ValueError("A dsn string must be provided.")
if not query:
raise ValueError("A query string must be provided.")
if not dsn and self.dsn_secret:
dsn = Secret(self.dsn_secret).get()
if not user and self.user_secret:
user = Secret(self.user_secret).get()
if not password and self.password_secret:
password = Secret(self.password_secret).get()
con = pyexasol.connect(
dsn=dsn,
user=user,
password=password,
autocommit=autocommit,
**kwargs,
)
# try to execute query
# context manager automatically rolls back failed transactions
with con as db:
db.execute(query, query_params)
if not autocommit:
if commit:
con.commit()
else:
con.rollback()
return
def token(self):
return str(Secret(self._token_secret).get())
def run(
self,
user: str = "",
password: str = "",
dsn: str = "",
destination: Union[str, Path] = None,
query_or_table: Union[str, tuple] = None,
query_params: dict = None,
export_params: dict = None,
**kwargs,
):
"""
Task run method. Executes a query against Postgres database.
Args:
- user (str, optional): user name used to authenticate. This overrides the
initial `user_secret` parameter (if set)
- password (str, optional): password used to authenticate. This overrides the
initial `user_secret` parameter (if set)
- dsn (str, optional): dsn string of the database (server:port)
This overrides the initial `user_secret` parameter (if set)
- destination ([str, Path], optional): Path to file or file-like object
- query_or_table (str, optional): SQL query or table for export
could be:
1. SELECT * FROM S.T
2. tablename
3. (schemaname, tablename)
- query_params (dict, optional): Values for SQL query placeholders
- export_params (dict, optional): custom parameters for EXPORT query
- **kwargs (dict, optional): additional connection parameter (connection_timeout...)
Returns:
- Nothing
Raises:
- ValueError: if dsn string is not provided
- ValueError: if destination is not provided
- ValueError: if no query or table are provided
- Exa*Error: multiple exceptions raised from the underlying pyexasol package
(e.g. ExaQueryError, ExaAuthError..)
"""
if not dsn and not self.dsn_secret:
raise ValueError("A dsn string must be provided.")
if not destination:
raise ValueError("A destination must be provided.")
if not query_or_table:
raise ValueError("A query or a table must be provided.")
if not dsn and self.dsn_secret:
dsn = Secret(self.dsn_secret).get()
if not user and self.user_secret:
user = Secret(self.user_secret).get()
if not password and self.password_secret:
password = Secret(self.password_secret).get()
con = pyexasol.connect(
dsn=dsn,
user=user,
password=password,
**kwargs,
)
# try to execute query
# context manager automatically rolls back failed transactions
with con as db:
db.export_to_file(
destination,
query_or_table,
query_params,
export_params,
)
return
def run(
self,
user: str = "",
password: str = "",
dsn: str = "",
target_schema: str = None,
target_table: str = None,
data: Iterable = None,
import_params: dict = None,
autocommit: bool = False,
commit: bool = True,
**kwargs,
):
"""
Task run method. Executes a query against Postgres database.
Args:
- user (str, optional): user name used to authenticate. This overrides the
initial `user_secret` parameter (if set)
- password (str, optional): password used to authenticate. This overrides the
initial `user_secret` parameter (if set)
- dsn (str, optional): dsn string of the database (server:port)
This overrides the initial `user_secret` parameter (if set)
- target_schema (str, optional): target schema for importing data
- target_table (str, optional): target table for importing data
- data (Iterable, optional): an iterable which holds the import data
- import_params (dict, optional): custom parameters for IMPORT query
- autocommit (bool, optional): turn autocommit on or off (default: False)
- commit (bool, optional): set to True to commit transaction, defaults to false
(only necessary if autocommit = False)
- **kwargs (dict, optional): additional connection parameter (connection_timeout...)
Returns:
- Nothing
Raises:
- ValueError: if dsn string is not provided
- ValueError: if `data` is not provided or is empty
- ValueError: if `target_table` is not provided
- Exa*Error: multiple exceptions raised from the underlying pyexasol package
(e.g. ExaQueryError, ExaAuthError..)
"""
if not dsn and not self.dsn_secret:
raise ValueError("A dsn string must be provided.")
if not data or len(data) == 0:
raise ValueError("Import Data must be provided.")
if not target_table:
raise ValueError("Target table must be provided.")
if not dsn and self.dsn_secret:
dsn = Secret(self.dsn_secret).get()
if not user and self.user_secret:
user = Secret(self.user_secret).get()
if not password and self.password_secret:
password = Secret(self.password_secret).get()
if not target_schema:
target = target_table
else:
target = (target_schema, target_table)
con = pyexasol.connect(
dsn=dsn,
user=user,
password=password,
autocommit=autocommit,
**kwargs,
)
# try to execute query
# context manager automatically rolls back failed transactions
with con as db:
db.import_from_iterable(data, target, import_params)
if not autocommit:
if commit:
con.commit()
else:
con.rollback()
return
def run(
self,
user: str = "",
password: str = "",
dsn: str = "",
fetch: str = "one",
fetch_size: int = 10,
query: str = None,
query_params: dict = None,
**kwargs,
):
"""
Task run method. Executes a query against Exasol database and fetches results.
Args:
- user (str, optional): user name used to authenticate. This overrides the
initial `user_secret` parameter (if set)
- password (str, optional): password used to authenticate. This overrides the
initial `user_secret` parameter (if set)
- dsn (str, optional): dsn string of the database (server:port)
This overrides the initial `user_secret` parameter (if set)
- fetch (str, optional): one of "one" "many" "val" or "all", used to determine how many
results to fetch from executed query
- fetch_size (int, optional): if fetch = 'many', determines the number of results
to fetch, defaults to 10
- query (str, optional): query to execute against database
- query_params (dict, optional): Values for SQL query placeholders
- **kwargs (dict, optional): additional connection parameter
(autocommit, connection_timeout...)
Returns:
- records (None, str, tuple, list of tuples, dict, or list of dicts):
records from provided query
Raises:
- ValueError: if dsn string is not provided
- ValueError: if query parameter is None or a blank string
- Exa*Error: multiple exceptions raised from the underlying pyexasol package
(e.g. ExaQueryError, ExaAuthError..)
"""
if not dsn and not self.dsn_secret:
raise ValueError("A dsn string must be provided.")
if not query:
raise ValueError("A query string must be provided.")
if fetch not in {"one", "many", "val", "all"}:
raise ValueError(
"The 'fetch' parameter must be one of the following - ('one', 'many', 'val', 'all')."
)
if not dsn and self.dsn_secret:
dsn = Secret(self.dsn_secret).get()
if not user and self.user_secret:
user = Secret(self.user_secret).get()
if not password and self.password_secret:
password = Secret(self.password_secret).get()
con = pyexasol.connect(
dsn=dsn,
user=user,
password=password,
**kwargs,
)
# try to execute query
# context manager automatically rolls back failed transactions
with con as db:
query = db.execute(query, query_params)
if fetch == "all":
return query.fetchall()
elif fetch == "many":
return query.fetchmany(fetch_size)
elif fetch == "val":
return query.fetchval()
else:
return query.fetchone()
def run(
self,
subject: str = None,
msg: str = None,
email_to: str = None,
email_from: str = None,
smtp_server: str = None,
smtp_port: int = None,
smtp_type: str = None,
msg_plain: str = None,
email_to_cc: str = None,
email_to_bcc: str = None,
attachments: List[str] = None,
) -> None:
username = cast(str, Secret("EMAIL_USERNAME").get())
password = cast(str, Secret("EMAIL_PASSWORD").get())
message = MIMEMultipart()
message["Subject"] = subject
message["From"] = email_from
message["To"] = email_to
if email_to_cc:
message["Cc"] = email_to_cc
if email_to_bcc:
message["Bcc"] = email_to_bcc
# First add the message in plain text, then the HTML version. Email clients try to render
# the last part first
if msg_plain:
message.attach(MIMEText(msg_plain, "plain"))
if msg:
message.attach(MIMEText(msg, "html"))
for filepath in attachments:
with open(filepath, "rb") as attachment:
part = MIMEBase("application", "octet-stream")
part.set_payload(attachment.read())
encoders.encode_base64(part)
filename = os.path.basename(filepath)
part.add_header(
"Content-Disposition",
f"attachment; filename= {filename}",
)
part.add_header("Content-ID", "<*****@*****.**>")
message.attach(part)
context = ssl.create_default_context()
if smtp_type == "SSL":
server = smtplib.SMTP_SSL(smtp_server, smtp_port, context=context)
elif smtp_type == "STARTTLS":
server = smtplib.SMTP(smtp_server, smtp_port)
server.starttls(context=context)
else:
raise ValueError(
f"{smtp_type} is an unsupported value for smtp_type")
server.login(username, password)
try:
server.send_message(message)
finally:
server.quit()
def run(
self,
repo: str = None,
base: str = None,
branch_name: str = None,
token: str = None,
) -> dict:
"""
Run method for this Task. Invoked by calling this Task after initialization within a Flow context,
or by using `Task.bind`.
Args:
- repo (str, optional): the name of the repository to open the issue in; must be provided in the
form `organization/repo_name`; defaults to the one provided at initialization
- base (str, optional): the name of the branch you want to branch off; if not provided here,
defaults to the one set at initialization
- branch_name (str, optional): the name of the new branch; if not provided here, defaults to
the one set at initialization
- token (str): a GitHub access token
Raises:
- ValueError: if a `repo` or `branch_name` was never provided, or if the base branch wasn't found
- HTTPError: if the GET request returns a non-200 status code
Returns:
- dict: dictionary of the response (includes commit hash, etc.)
"""
if branch_name is None:
raise ValueError("A branch name must be provided.")
if repo is None:
raise ValueError("A GitHub repository must be provided.")
## prepare the request
if token is None:
warnings.warn(
"The `token` argument is deprecated. Use a `Secret` task "
"to pass the credentials value at runtime instead.",
UserWarning,
)
token = Secret(self.token_secret).get()
url = "https://api.github.com/repos/{}/git/refs".format(repo)
headers = {
"AUTHORIZATION": "token {}".format(token),
"Accept": "application/vnd.github.v3+json",
}
## gather branch information
resp = requests.get(url + "/heads", headers=headers)
resp.raise_for_status()
branch_data = resp.json()
commit_sha = None
for branch in branch_data:
if branch.get("ref") == "refs/heads/{}".format(base):
commit_sha = branch.get("object", {}).get("sha")
break
if commit_sha is None:
raise ValueError("Base branch {} not found.".format(base))
## create new branch
new_branch = {
"ref": "refs/heads/{}".format(branch_name),
"sha": commit_sha
}
resp = requests.post(url, headers=headers, json=new_branch)
resp.raise_for_status()
return resp.json()
