def get_metastore_params_template(cls):
return StructFormField(
connection_string=FormField(
required=True,
description="Put your JDBC connection string here",
regex=
"^(?:jdbc:)?hive2:\\/\\/([\\w.-]+(?:\\:\\d+)?(?:,[\\w.-]+(?:\\:\\d+)?)*)\\/(\\w*)((?:;[\\w.-]+=[\\w.-]+)*)(\\?[\\w.-]+=[\\w.-]+(?:;[\\w.-]+=[\\w.-]+)*)?(\\#[\\w.-]+=[\\w.-]+(?:;[\\w.-]+=[\\w.-]+)*)?$", # noqa: E501
helper="""
<p>
Format
jdbc:hive2://<host1>:<port1>,<host2>:<port2>/dbName;sess_var_list?hive_conf_list#hive_var_list
</p>
<p>Currently support zookeeper in session var, and will pass any conf variables to HS2</p>
<p>See
<a href="https://cwiki.apache.org/confluence/display/Hive/HiveServer2+Clients#HiveServer2Clients-JDBC">
here
</a> for more details.
</p>""",
),
username=FormField(regex="\\w+"),
password=FormField(hidden=True),
hms_connection=ExpandableFormField(
of=FormField(
required=True,
description="Put url to hive metastore server here"),
min=1,
),
)
def test_null_field(self):
# Code coverage test cases
# Null case test
self.assertEqual(
validate_form(FormField(required=True), None),
(False, "Required field is missing"),
)
self.assertEqual(validate_form(FormField(), None), (True, ""))
def get_metastore_params_template(cls):
return StructFormField(
catalog_id=FormField(
required=True,
description="Enter the Glue Data Catalog ID",
regex=r"^\d{12}$",
),
region=FormField(required=True,
description="Enter the AWS Region"),
load_partitions=load_partitions_field,
)
def test_bool_field(self):
self.assertEqual(
validate_form(FormField(field_type=FormFieldType.Boolean), "123"),
(False, "Field value is not a boolean"),
)
self.assertEqual(
validate_form(FormField(field_type=FormFieldType.Boolean), 123),
(False, "Field value is not a boolean"),
)
self.assertEqual(
validate_form(FormField(field_type=FormFieldType.Boolean), True),
(True, ""))
def test_number_field(self):
self.assertEqual(
validate_form(FormField(field_type=FormFieldType.Number), "123"),
(False, "Field value is not a number"),
)
self.assertEqual(
validate_form(FormField(field_type=FormFieldType.Number), 123),
(True, ""))
self.assertEqual(
validate_form(FormField(field_type=FormFieldType.Number), 123.123),
(True, ""),
)
def test_string_field(self):
# String Tests
self.assertEqual(validate_form(FormField(), 123),
(False, "Field value is not a string"))
self.assertEqual(validate_form(FormField(), "123"), (True, ""))
self.assertEqual(
validate_form(FormField(regex="^[a-z]+$"), "querybook2"),
(False, "Field value does not match regex"),
)
self.assertEqual(
validate_form(FormField(regex="^[a-z]+$"), "querybook"),
(True, ""))
def export_form(self):
return StructFormField(
sheet_url=FormField(
description="Optional, if not provided a new sheet will be created."
),
worksheet_title=FormField(
description='Defaults to "Sheet1"',
helper="Title of the worksheet, if not found then a sheet will be created",
),
start_cell=FormField(
description="The top left cell position where data will be filled. Defaults to A1",
regex="^[A-Z]{1,3}[1-9][0-9]*$",
),
)
def get_metastore_params_template(cls):
return StructFormField(
catalog_id=FormField(
required=True,
description="Enter the Glue Data Catalog ID",
regex=r"^\d{12}$",
),
region=FormField(required=True,
description="Enter the AWS Region"),
load_partitions=FormField(
required=False,
field_type=FormFieldType.Boolean,
helper=
"""In case your data catalog is large, loading all partitions for all tables can be quite time consuming.
Skipping partition information can reduce your metastore refresh latency
""",
),
)
def get_metastore_params_template(cls):
return StructFormField(
hms_connection=ExpandableFormField(
of=FormField(
required=True,
description="Put url to hive metastore server here",
field_type=FormFieldType.String,
),
min=1,
),
load_partitions=load_partitions_field,
)
def test_dict_field(self):
form = StructFormField(
name=FormField(),
phone_numbers=ExpandableFormField(of=FormField(), min=1, max=2),
)
self.assertEqual(
validate_form(form, "123"),
(False, "Field value is not a dictionary"),
)
self.assertEqual(
validate_form(form, {
"phone_numbers": [1234],
"name": "bob"
}),
(False, "Field value is not a string"),
)
self.assertEqual(
validate_form(form, {
"phone_numbers": ["1234"] * 3,
"name": "bob"
}),
(False, "Field value more than allowed length"),
)
self.assertEqual(
validate_form(form, {
"phone_numbers": ["1234"],
"name": "bob"
}),
(True, ""),
)
self.assertEqual(
validate_form(
form,
{
"phone_numbers": ["1234"],
},
),
(True, ""),
)
def test_array_field(self):
form = ExpandableFormField(of=FormField(), min=2, max=4)
self.assertEqual(
validate_form(form, "123"),
(False, "Field value is not an array"),
)
self.assertEqual(
validate_form(form, ["123"]),
(False, "Field value less than allowed length"),
)
self.assertEqual(
validate_form(form, ["123"] * 5),
(False, "Field value more than allowed length"),
)
self.assertEqual(
validate_form(form, ["123", "123", 123]),
(False, "Field value is not a string"),
)
self.assertEqual(validate_form(form, ["123", "456", "789"]),
(True, ""))
def get_metastore_params_template(cls):
return StructFormField(connection_string=FormField(
required=True,
description="Put sqlalchemy connection string here"), )
from lib.form import FormField, FormFieldType
load_partitions_field = FormField(
required=False,
field_type=FormFieldType.Boolean,
helper=
"""In case your data catalog is large, loading all partitions for all tables can be quite time consuming.
Skipping partition information can reduce your metastore refresh latency
""",
)
from lib.form import FormField, StructFormField, FormFieldType, ExpandableFormField
hive_executor_template = StructFormField(
hive_resource_manager=FormField(
description="Provide resource manager link here to provide insights"),
connection_string=FormField(
required=True,
description="Put your JDBC connection string here",
regex=
"^(?:jdbc:)?hive2:\\/\\/([\\w.-]+(?:\\:\\d+)?(?:,[\\w.-]+(?:\\:\\d+)?)*)\\/(\\w*)((?:;[\\w.-]+=[\\w.-]+)*)(\\?[\\w.-]+=[\\w.-]+(?:;[\\w.-]+=[\\w.-]+)*)?(\\#[\\w.-]+=[\\w.-]+(?:;[\\w.-]+=[\\w.-]+)*)?$", # noqa: E501
helper="""
<p>
Format
jdbc:hive2://<host1>:<port1>,<host2>:<port2>/dbName;sess_var_list?hive_conf_list#hive_var_list
</p>
<p>Currently support zookeeper in session var, and will pass any conf variables to HS2</p>
<p>See [here](https://cwiki.apache.org/confluence/display/Hive/HiveServer2+Clients#HiveServer2Clients-JDBC) for more details.
</p>""",
),
username=FormField(regex="\\w+"),
password=FormField(hidden=True),
impersonate=FormField(field_type=FormFieldType.Boolean),
)
presto_executor_template = StructFormField(
connection_string=FormField(
required=True,
regex=
"^(?:jdbc:)?presto:\\/\\/([\\w.-]+(?:\\:\\d+)?(?:,[\\w.-]+(?:\\:\\d+)?)*)(\\/\\w+)?(\\/\\w+)?(\\?[\\w.-]+=[\\w.-]+(?:&[\\w.-]+=[\\w.-]+)*)?$", # noqa: E501
helper="""
<p>Format jdbc:presto://<host:port>/<catalog>/<schema>?presto_conf_list</p>
def dag_exporter_meta(self):
return StructFormField(
title=FormField(description="dag title"),
description=FormField(description="dag description"),
)
def get_metastore_params_template(cls):
return ExpandableFormField(
of=FormField(required=True,
description="Put url to hive metastore server here"),
min=1,
)
