def validate_multiple(self, field_mappings: FieldMappings) -> Dict[str, Dict]:
"""Returns map of field_name -> failures. If map is empty, the field mapping approvals are valid."""
return ge_results_to_failure_map(
{
field_name: self._get_expectations(field_mapping).validate()
for field_name, field_mapping in field_mappings.items()
}
)
def _get_batch_clear_mappings_body(self, field_mappings: FieldMappings):
"""
Provides the body for batch clearing multiple ranges in a Google Sheet
for the provided field mappings.
"""
ranges: List[str] = [
f"{field_name}!A:C" for field_name, _ in field_mappings.items()
]
return {"ranges": ranges}
def _get_batch_update_mappings_body(self, field_mappings: FieldMappings):
"""
Provides the body for batch updating multiple ranges in a Google Sheet
with provided field mappings.
"""
data: List[Dict] = [
{
"range": f"{field_name}!A:C",
"values": [common.COLUMN_NAMES]
+ field_mapping.get_field_mapping_df().values.tolist(),
}
for field_name, field_mapping in field_mappings.items()
]
return {"valueInputOption": "RAW", "data": data}
def resolve_mappings(
new_mappings: FieldMappings,
source_mappings: FieldMappings,
overwrite: bool = False,
remove_unapproved_source_mappings: bool = True,
) -> FieldMappings:
"""
Resolves a set of source and new FieldMappings.
"""
resolved_mappings = {}
source_fields = set(source_mappings.keys())
new_fields = set(new_mappings.keys())
for field in source_fields | new_fields:
if field in source_fields:
source_mapping = source_mappings[field]
else:
source_mapping = FieldMapping.from_dict({})
if field in new_fields:
new_mapping = new_mappings[field]
else:
new_mapping = FieldMapping.from_dict({})
resolved_mapping = FieldMappingResolver._resolve_mapping(
new_mapping,
source_mapping,
overwrite,
remove_unapproved_source_mappings,
)
if not resolved_mapping.is_empty():
resolved_mappings[field] = resolved_mapping
return resolved_mappings
def write_field_mappings_local(field_mappings: FieldMappings, config_location: str):
"""Writes all field mappings as a .csv to a provided directory."""
dirname: str = os.path.dirname(
config_location
) # TODO: are we sure we want dirname()?
# Example: os.path.dirname(myparent/mydir) -> "myparent"
if not os.path.exists(dirname):
logging.error(
"Invalid directory name provided: '%s' may not exist.", dirname
)
for field_name, field_mapping in field_mappings.items():
field_mapping_dict = field_mapping.get_field_mapping_dict()
if not field_mapping_dict:
continue
mapping_filename: str = common.get_field_mapping_filename(
field_name, dirname
)
FieldMappingWriter.write_field_mapping_local(
field_mapping_dict, mapping_filename
)
def write_field_mappings_drive(
self, field_mappings: FieldMappings, account_info: Dict, spreadsheet_id: str
):
"""
Writes all field mappings to a Google Sheet. Creates new sheets when
required.
Sets data validations on the columns and auto resizes column widths.
"""
service: Resource = drive.get_google_sheets_service(account_info)
# Get information on the existing sheets
existing_sheets: List[SheetInfo] = drive.get_sheets_for_spreadsheet(
service, spreadsheet_id
)
# Add sheets that don't exist yet
existing_fields = drive.get_sheet_titles_from_sheets(existing_sheets)
unwritten_fields: List[str] = [
field for field in field_mappings.keys() if field not in existing_fields
]
add_sheets_response = drive.add_sheets(
service, unwritten_fields, spreadsheet_id
)
# Clear sheets
batch_clear_mappings_body = self._get_batch_clear_mappings_body(field_mappings)
drive.value_batch_clear(service, batch_clear_mappings_body, spreadsheet_id)
# Update field mappings
batch_update_mappings_body = self._get_batch_update_mappings_body(
field_mappings
)
drive.value_batch_update(service, batch_update_mappings_body, spreadsheet_id)
# Get information for sheets that were just added
if add_sheets_response:
new_sheets: List[SheetInfo] = [
{
"title": reply["addSheet"]["properties"]["title"],
"sheetId": reply["addSheet"]["properties"]["sheetId"],
}
for reply in add_sheets_response["replies"]
]
else:
new_sheets: List[SheetInfo] = []
all_valid_sheets = [
sheet
for sheet in existing_sheets + new_sheets
if sheet["title"] in self.table_schema.field_names
]
# Update field level validations (allowed enums and approved options)
# and auto resize columns in spreadsheets
data_validation_requests = self._get_data_validation_requests(all_valid_sheets)
auto_resize_requests = self._get_auto_resize_requests(all_valid_sheets)
requests = data_validation_requests + auto_resize_requests
if requests:
body = {"requests": requests}
drive.batch_update(service, body, spreadsheet_id)