def write_to_hdfs(rows: List[Tuple[str, str]]):
conn: Connection = Connection.get_connection_from_secrets('local_hdfs')
uri = conn.get_uri()
pat = re.compile("http://(\w+(:\w+)?)?@")
print(conn.get_uri())
uri = pat.sub("http://", uri)
print(uri)
print(conn.login)
client = InsecureClient(uri, user=conn.login)
sch = avro.schema.make_avsc_object({
'type':'record',
'name':'Video',
'fields': [
{'type': {'type': 'string', 'avro.java.string': 'String'}, 'name': 'title'},
{'type': ["null", {'type': 'string', 'avro.java.string': 'String'}], 'name': 'description'},
]
})
local_file_name = 'videos.avro'
writer = DataFileWriter(open(local_file_name, "wb"), DatumWriter(), sch)
for row in rows:
print(row)
writer.append({"title":row[0], "description":row[1]})
writer.close()
client.upload('/tmp/videos.avro', local_file_name)
def get(self, key: str, default_conn: Any = None) -> Any:
from airflow.exceptions import AirflowNotFoundException
from airflow.models.connection import Connection
try:
return Connection.get_connection_from_secrets(key)
except AirflowNotFoundException:
return default_conn
def load_from_cassandra() -> List[Tuple[str, str]]:
conn: Connection = Connection.get_connection_from_secrets('local_cassandra')
auth_provider = PlainTextAuthProvider(username=conn.login, password=conn.password)
cluster: Cluster = Cluster([conn.host], conn.port, auth_provider=auth_provider)
session: Session = cluster.connect(conn.schema)
rows: ResultSet = session.execute("SELECT title, description FROM videos")
result = list(map(lambda row: (row[0], row[1]), rows))
print(result)
return result
def get_connection(cls, conn_id: str) -> "Connection":
"""
Get connection, given connection id.
:param conn_id: connection id
:return: connection
"""
from airflow.models.connection import Connection
conn = Connection.get_connection_from_secrets(conn_id)
log.info("Using connection ID '%s' for task execution.", conn.conn_id)
return conn
def get_connection(cls, conn_id: str) -> Connection:
"""
Get connection, given connection id.
:param conn_id: connection id
:return: connection
"""
conn = Connection.get_connection_from_secrets(conn_id)
if conn.host:
log.info(
"Using connection to: id: %s. Host: %s, Port: %s, Schema: %s, Login: %s, Password: %s, "
"extra: %s",
conn.conn_id,
conn.host,
conn.port,
conn.schema,
conn.login,
"XXXXXXXX" if conn.password else None,
"XXXXXXXX" if conn.extra_dejson else None,
)
return conn
def get_connection(cls, conn_id: str) -> "Connection":
"""
Get connection, given connection id.
:param conn_id: connection id
:return: connection
"""
from airflow.models.connection import Connection
conn = Connection.get_connection_from_secrets(conn_id)
if conn.host:
log.info(
"Using connection to: id: %s. Host: %s, Port: %s, Schema: %s, Login: %s, Password: %s, "
"extra: %s",
conn.conn_id,
conn.host,
conn.port,
conn.schema,
conn.login,
conn.password,
conn.extra_dejson,
)
return conn
def get_connection(cls, conn_id: str) -> "Connection":
"""
Get connection, given connection id.
:param conn_id: connection id
:return: connection
"""
from airflow.models.connection import Connection
conn = Connection.get_connection_from_secrets(conn_id)
log.info("Using connection ID '%s' for task execution.", conn.conn_id)
log.debug(
"Connection details for '%s':: Host: %s, Port: %s, Schema: %s, Login: %s, Password: %s, "
"Extra: %s",
conn.conn_id,
conn.host,
conn.port,
conn.schema,
conn.login,
redact(conn.password),
redact(conn.extra_dejson),
)
return conn
def __getattr__(self, key: str) -> Any:
from airflow.models.connection import Connection
self.var = Connection.get_connection_from_secrets(key)
return self.var
def does_connection_exist(conn_id: str) -> bool:
try:
Connection.get_connection_from_secrets(conn_id=conn_id)
return True
except AirflowNotFoundException:
return False
