def __create_tag_template(cls):
template = datacatalog.TagTemplate()
tag_template_id = 'template'
template.name = 'template name'
template.display_name = 'template display name'
bool_field = datacatalog.TagTemplateField()
bool_field.type.primitive_type = cls.__BOOL_TYPE
template.fields['bool-field'] = bool_field
double_field = datacatalog.TagTemplateField()
double_field.type.primitive_type = cls.__DOUBLE_TYPE
template.fields['double-field'] = double_field
string_field = datacatalog.TagTemplateField()
string_field.type.primitive_type = cls.__STRING_TYPE
template.fields['string-field'] = string_field
timestamp_field = datacatalog.TagTemplateField()
timestamp_field.type.primitive_type = cls.__TIMESTAMP_TYPE
template.fields['timestamp-field'] = timestamp_field
enum_value = datacatalog.FieldType.EnumType.EnumValue()
enum_value.display_name = 'Test ENUM Value'
enum_field = datacatalog.TagTemplateField()
enum_field.type.enum_type.allowed_values.append(enum_value)
template.fields['enum-field'] = enum_field
return tag_template_id, template
def __set_tag_template_field(self, attribute_def, enum_types_dict, name,
tag_template):
formatted_name = attr_normalizer.DataCatalogAttributeNormalizer.\
format_name(name)
type_name = attribute_def['typeName']
enum_type = enum_types_dict.get(type_name)
field = datacatalog.TagTemplateField()
field.display_name = name
if type_name in constant.DATACATALOG_TARGET_PRIMITIVE_TYPES:
if type_name in constant.DATACATALOG_TARGET_DOUBLE_TYPE:
target_type = self.__DOUBLE_TYPE
elif type_name in constant.DATACATALOG_TARGET_BOOLEAN_TYPE:
target_type = self.__BOOL_TYPE
else:
target_type = self.__STRING_TYPE
field.type.primitive_type = target_type
elif enum_type:
enum_element_defs = enum_type['data']['elementDefs']
for enum_element_def in enum_element_defs:
enum_value = datacatalog.FieldType.EnumType.EnumValue()
enum_value.display_name = enum_element_def['value']
field.type.enum_type.allowed_values.append(enum_value)
else:
# String is the default type.
field.type.primitive_type = self.__STRING_TYPE
tag_template.fields[formatted_name] = field
def _add_primitive_type_field(cls, tag_template, field_id, field_type,
display_name):
field = datacatalog.TagTemplateField()
field.type.primitive_type = field_type
field.display_name = display_name
tag_template.fields[field_id] = field
def __make_enum_type_template_field(values):
field = datacatalog.TagTemplateField()
for value in values:
enum_value = datacatalog.FieldType.EnumType.EnumValue()
enum_value.display_name = value
field.type_.enum_type.allowed_values.append(enum_value)
return field
def _add_enum_type_field(cls, tag_template, field_id, values,
display_name):
field = datacatalog.TagTemplateField()
for value in values:
enum_value = datacatalog.FieldType.EnumType.EnumValue()
enum_value.display_name = value
field.type.enum_type.allowed_values.append(enum_value)
field.display_name = display_name
tag_template.fields[field_id] = field
def _add_primitive_type_field(cls,
tag_template,
field_id,
field_type,
display_name,
is_required=False,
order=None):
field = datacatalog.TagTemplateField()
field.type.primitive_type = field_type
field.display_name = display_name
field.is_required = is_required
field.order = order
tag_template.fields[field_id] = field
def create_tag_template_field(self, template_name, field_id, display_name,
enum_values):
"""Add field to a Tag Template."""
field = datacatalog.TagTemplateField()
field.display_name = display_name
for enum_value in enum_values:
value = datacatalog.FieldType.EnumType.EnumValue()
value.display_name = enum_value['display_name']
field.type_.enum_type.allowed_values.append(value)
return self.__datacatalog.create_tag_template_field(
parent=template_name,
tag_template_field_id=field_id,
tag_template_field=field)
def _add_enum_type_field(cls,
tag_template,
field_id,
values,
display_name,
is_required=False,
order=None):
field = datacatalog.TagTemplateField()
for value in values:
enum_value = datacatalog.FieldType.EnumType.EnumValue()
enum_value.display_name = value
field.type.enum_type.allowed_values.append(enum_value)
field.display_name = display_name
field.is_required = is_required
field.order = order
tag_template.fields[field_id] = field
def create_tag_template(self, project_id, template_id, display_name,
primitive_fields_descriptors):
"""Create a Tag Template."""
location = self.__datacatalog.common_location_path(
project_id, 'us-central1')
tag_template = datacatalog.TagTemplate()
tag_template.display_name = display_name
for descriptor in primitive_fields_descriptors:
field = datacatalog.TagTemplateField()
field.display_name = descriptor['display_name']
field.type_.primitive_type = descriptor['primitive_type']
tag_template.fields[descriptor['id']] = field
return self.__datacatalog.create_tag_template(
parent=location,
tag_template_id=template_id,
tag_template=tag_template)
def create_tag_template(self,
project_id,
template_id,
display_name,
fields_descriptors,
enums_names=None):
"""Create a Tag Template."""
location = datacatalog.DataCatalogClient.common_location_path(
project_id, _CLOUD_PLATFORM_REGION)
tag_template = datacatalog.TagTemplate()
tag_template.display_name = display_name
for descriptor in fields_descriptors:
field = datacatalog.TagTemplateField()
field.display_name = descriptor[1]
field_id = descriptor[0]
field_type = descriptor[2]
if not field_type == _DATA_CATALOG_ENUM_TYPE:
field.type_.primitive_type = datacatalog.FieldType.PrimitiveType[
field_type]
else:
for enum_name in enums_names[field_id]:
enum_value = datacatalog.FieldType.EnumType.EnumValue()
enum_value.display_name = enum_name
field.type_.enum_type.allowed_values.append(enum_value)
tag_template.fields[field_id] = field
created_tag_template = self.__datacatalog.create_tag_template(
parent=location,
tag_template_id=template_id,
tag_template=tag_template)
logging.info(f'===> Template created: {created_tag_template.name}')
def create_template(project_id, region, yaml_file):
dc_client = DataCatalogClient()
tag_template = datacatalog.TagTemplate()
with open(yaml_file) as file:
file_contents = yaml.full_load(file)
template_contents = file_contents.get("template")[0]
for k, v in template_contents.items():
print(k + "->" + str(v))
if k == 'name':
tag_template_id = v
if k == 'display_name':
tag_template.display_name = v
if k == 'fields':
fields = v
for field in fields:
field_id = None
datatype = None
enum_values = None
display_name = None
required = False
for fname, fval in field.items():
print(fname + "->" + str(fval))
if fname == "field":
field_id = fval
if fname == "type":
datatype = fval
if fname == "values":
enum_values = fval
if fname == "display":
display = fval
if fname == "required":
required = fval
if fname == "order":
order = fval
if datatype.lower() == "enum":
field = datacatalog.TagTemplateField()
enum_list = enum_values.split("|")
for value in enum_list:
enum_value = datacatalog.FieldType.EnumType.EnumValue(
)
enum_value.display_name = value
field.type.enum_type.allowed_values.append(
enum_value)
field.display_name = display
field.is_required = required
field.order = order
tag_template.fields[field_id] = field
elif datatype.lower() == "bool":
field = datacatalog.TagTemplateField()
field.type.primitive_type = datacatalog.FieldType.PrimitiveType.BOOL
field.display_name = display
field.is_required = required
field.order = order
tag_template.fields[field_id] = field
elif datatype.lower() == "string":
field = datacatalog.TagTemplateField()
field.type.primitive_type = datacatalog.FieldType.PrimitiveType.STRING
field.display_name = display
field.is_required = required
field.order = order
tag_template.fields[field_id] = field
elif datatype.lower() == "double":
field = datacatalog.TagTemplateField()
field.type.primitive_type = datacatalog.FieldType.PrimitiveType.DOUBLE
field.display_name = display
field.is_required = required
field.order = order
tag_template.fields[field_id] = field
elif datatype.lower() == "timestamp":
field = datacatalog.TagTemplateField()
field.type.primitive_type = datacatalog.FieldType.PrimitiveType.TIMESTAMP
field.display_name = display
field.is_required = required
field.order = order
tag_template.fields[field_id] = field
created_template = dc_client.create_tag_template(
parent=f'projects/{project_id}/locations/{region}',
tag_template_id=tag_template_id,
tag_template=tag_template)
return created_template
def make_primitive_type_template_field(
primitive_type: FieldType.PrimitiveType):
field = datacatalog.TagTemplateField()
field.type_.primitive_type = primitive_type
return field