def get_message_level(
val_rule: str,
sg: SchemaGenerator,
attribute_name: str,
) -> str:
"""
Purpose:
Determine whether an error or warning message should be logged and displayed
Types of error/warning included:
- recommended - Raised when an attribute is empty and recommended but not required.
- unique - Raised when attribute values are not unique.
- protectAges - Raised when an attribute contains ages below 18YO or over 90YO that should be censored.
Input:
val_rule: str, defined in the schema.
sg: schemaGenerator object
attribute_name: str, attribute being validated
Returns:
'error' or 'warning'
"""
rule_parts = val_rule.split(" ")
#See if the node is required, if it is and the column is missing then a requirement error will be raised later; no error or waring logged here if recommended and required but missing
if val_rule.startswith('recommended') and sg.is_node_required(
node_display_name=attribute_name):
level = None
#if not required, use the message level specified in the rule
elif rule_parts[-1].lower() == 'error':
level = 'error'
elif rule_parts[-1].lower() == 'warning':
level = 'warning'
#if no level specified, the default level is warning
else:
level = 'warning'
return level
class ManifestGenerator(object):
def __init__(
self,
path_to_json_ld:
str, # JSON-LD file to be used for generating the manifest
title: str = None, # manifest sheet title
root: str = None,
additional_metadata: Dict = None,
oauth: bool = True,
use_annotations: bool = False,
) -> None:
"""TODO: read in a config file instead of hardcoding paths to credential files...
"""
if oauth:
# if user wants to use OAuth for Google authentication
# use credentials.json and create token.pickle file
services_creds = build_credentials()
else:
# if not oauth then use service account credentials
services_creds = build_service_account_creds()
# google service for Sheet API
self.sheet_service = services_creds["sheet_service"]
# google service for Drive API
self.drive_service = services_creds["drive_service"]
# google service credentials object
self.creds = services_creds["creds"]
# schema root
self.root = root
# manifest title
self.title = title
if self.title is None:
self.title = f"{self.root} - Manifest"
# Whether to use existing annotations during manifest generation
self.use_annotations = use_annotations
# Warn about limited feature support for `use_annotations`
if self.use_annotations:
logger.warning(
"The `use_annotations` option is currently only supported "
"when there is no manifest file for the dataset in question.")
# SchemaGenerator() object
self.sg = SchemaGenerator(path_to_json_ld)
# additional metadata to add to manifest
self.additional_metadata = additional_metadata
# Determine whether current data type is file-based
is_file_based = False
if self.root:
is_file_based = "Filename" in self.sg.get_node_dependencies(
self.root)
self.is_file_based = is_file_based
def _attribute_to_letter(self, attribute, manifest_fields):
"""Map attribute to column letter in a google sheet
"""
# find index of attribute in manifest field
column_idx = manifest_fields.index(attribute)
# return the google sheet letter representation of the column index
return self._column_to_letter(column_idx)
def _column_to_letter(self, column):
"""Find google sheet letter representation of a column index integer
"""
character = chr(ord('A') + column % 26)
remainder = column // 26
if column >= 26:
return self._column_to_letter(remainder - 1) + character
else:
return character
def _columns_to_sheet_ranges(self, column_idxs):
"""map a set of column indexes to a set of Google sheet API ranges: each range includes exactly one column
"""
ranges = []
for column_idx in column_idxs:
col_range = {
"startColumnIndex": column_idx,
"endColumnIndex": column_idx + 1
}
ranges.append(col_range)
return ranges
def _column_to_cond_format_eq_rule(self,
column_idx: int,
condition_argument: str,
required: bool = False) -> dict:
"""Given a column index and an equality argument (e.g. one of valid values for the given column fields), generate a conditional formatting rule based on a custom formula encoding the logic:
'if a cell in column idx is equal to condition argument, then set specified formatting'
"""
col_letter = self._column_to_letter(column_idx)
if not required:
bg_color = CONFIG["style"]["google_manifest"].get(
"opt_bg_color", {
"red": 1.0,
"green": 1.0,
"blue": 0.9019,
})
else:
bg_color = CONFIG["style"]["google_manifest"].get(
"req_bg_color", {
"red": 0.9215,
"green": 0.9725,
"blue": 0.9803,
})
boolean_rule = {
"condition": {
"type":
"CUSTOM_FORMULA",
"values": [{
"userEnteredValue":
'=$' + col_letter + '1 = "' + condition_argument + '"'
}]
},
"format": {
'backgroundColor': bg_color
}
}
return boolean_rule
def _gdrive_copy_file(self, origin_file_id, copy_title):
"""Copy an existing file.
Args:
origin_file_id: ID of the origin file to copy.
copy_title: Title of the copy.
Returns:
The copied file if successful, None otherwise.
"""
copied_file = {'name': copy_title}
# return new copy sheet ID
return self.drive_service.files().copy(
fileId=origin_file_id, body=copied_file).execute()["id"]
def _create_empty_manifest_spreadsheet(self, title):
if CONFIG["style"]["google_manifest"]["master_template_id"]:
# if provided with a template manifest google sheet, use it
spreadsheet_id = self._gdrive_copy_file(
CONFIG["style"]["google_manifest"]["master_template_id"],
title)
else:
# if no template, create an empty spreadsheet
spreadsheet = self.sheet_service.spreadsheets().create(
body=spreadsheet, fields='spreadsheetId').execute()
spreadsheet_id = spreadsheet.get('spreadsheetId')
return spreadsheet_id
def _get_cell_borders(self, cell_range):
#set border style request
color = {
"red": 226.0 / 255.0,
"green": 227.0 / 255.0,
"blue": 227.0 / 255.0,
}
border_style_req = {
"updateBorders": {
"range": cell_range,
"top": {
"style": "SOLID",
"width": 2,
"color": color
},
"bottom": {
"style": "SOLID",
"width": 2,
"color": color
},
"left": {
"style": "SOLID",
"width": 2,
"color": color
},
"right": {
"style": "SOLID",
"width": 2,
"color": color
},
"innerHorizontal": {
"style": "SOLID",
"width": 2,
"color": color
},
"innerVertical": {
"style": "SOLID",
"width": 2,
"color": color
}
}
}
return border_style_req
def _set_permissions(self, fileId):
def callback(request_id, response, exception):
if exception:
# Handle error
logger.error(exception)
else:
logger.info(f"Permission Id: {response.get('id')}")
batch = self.drive_service.new_batch_http_request(callback=callback)
worldPermission = {'type': 'anyone', 'role': 'writer'}
batch.add(self.drive_service.permissions().create(
fileId=fileId,
body=worldPermission,
fields='id',
))
batch.execute()
def _get_column_data_validation_values(
self,
spreadsheet_id,
valid_values,
column_id,
validation_type="ONE_OF_LIST",
strict=True,
custom_ui=True,
input_message="Choose one from dropdown"):
# get valid values w/o google sheet header
values = [
valid_value["userEnteredValue"] for valid_value in valid_values
]
if validation_type == "ONE_OF_RANGE":
# store valid values explicitly in workbook at the provided range to use as validation values
target_col_letter = self._column_to_letter(column_id)
body = {"majorDimension": "COLUMNS", "values": [values]}
target_range = 'Sheet2!' + target_col_letter + '2:' + target_col_letter + str(
len(values) + 1)
valid_values = [{"userEnteredValue": "=" + target_range}]
response = self.sheet_service.spreadsheets().values().update(
spreadsheetId=spreadsheet_id,
range=target_range,
valueInputOption="RAW",
body=body).execute()
# setup validation data request body
validation_body = {
"requests": [{
'setDataValidation': {
'range': {
'startRowIndex': 1,
'startColumnIndex': column_id,
'endColumnIndex': column_id + 1,
},
'rule': {
'condition': {
'type': validation_type,
'values': valid_values
},
'inputMessage': input_message,
'strict': strict,
'showCustomUi': custom_ui
}
}
}]
}
return validation_body
def _get_valid_values_from_jsonschema_property(self,
prop: dict) -> List[str]:
"""Get valid values for a manifest attribute based on the corresponding
values of node's properties in JSONSchema
Args:
prop: node properties - jsonschema dictionary
Returns:
List of valid values
"""
if "enum" in prop:
return prop["enum"]
elif "items" in prop:
return prop["items"]["enum"]
else:
return []
def get_empty_manifest(self, json_schema_filepath=None):
# TODO: Refactor get_manifest method
# - abstract function for requirements gathering
# - abstract google sheet API requests as functions
# --- specifying row format
# --- setting valid values in dropdowns for columns/cells
# --- setting notes/comments to cells
spreadsheet_id = self._create_empty_manifest_spreadsheet(self.title)
if not json_schema_filepath:
# if no json schema is provided; there must be
# schema explorer defined for schema.org schema
# o.w. this will throw an error
# TODO: catch error
json_schema = self.sg.get_json_schema_requirements(
self.root, self.title)
else:
with open(json_schema_filepath) as jsonfile:
json_schema = json.load(jsonfile)
required_metadata_fields = {}
# gathering dependency requirements and corresponding allowed values constraints (i.e. valid values) for root node
for req in json_schema["properties"].keys():
required_metadata_fields[
req] = self._get_valid_values_from_jsonschema_property(
json_schema["properties"][req])
# the following line may not be needed
json_schema["properties"][req]["enum"] = required_metadata_fields[
req]
# gathering dependency requirements and allowed value constraints for conditional dependencies if any
if "allOf" in json_schema:
for conditional_reqs in json_schema["allOf"]:
if "required" in conditional_reqs["if"]:
for req in conditional_reqs["if"]["required"]:
if req in conditional_reqs["if"]["properties"]:
if not req in required_metadata_fields:
if req in json_schema["properties"]:
required_metadata_fields[
req] = self._get_valid_values_from_jsonschema_property(
json_schema["properties"][req])
else:
required_metadata_fields[
req] = self._get_valid_values_from_jsonschema_property(
conditional_reqs["if"]
["properties"][req])
for req in conditional_reqs["then"]["required"]:
if not req in required_metadata_fields:
if req in json_schema["properties"]:
required_metadata_fields[
req] = self._get_valid_values_from_jsonschema_property(
json_schema["properties"][req])
# if additional metadata is provided append columns (if those do not exist already)
if self.additional_metadata:
for column in self.additional_metadata.keys():
if not column in required_metadata_fields:
required_metadata_fields[column] = []
# if 'component' is in column set (see your input jsonld schema for definition of 'component', if the 'component' attribute is present), add the root node as an additional metadata component entry
if 'Component' in required_metadata_fields.keys():
# check if additional metadata has actually been instantiated in the constructor (it's optional)
# if not, instantiate it
if not self.additional_metadata:
self.additional_metadata = {}
self.additional_metadata['Component'] = [self.root]
# adding columns to manifest sheet
end_col = len(required_metadata_fields.keys())
end_col_letter = self._column_to_letter(end_col)
# order columns header (since they are generated based on a json schema, which is a dict)
ordered_metadata_fields = [list(required_metadata_fields.keys())]
ordered_metadata_fields[0] = self.sort_manifest_fields(
ordered_metadata_fields[0])
body = {"values": ordered_metadata_fields}
#determining columns range
end_col = len(required_metadata_fields.keys())
end_col_letter = self._column_to_letter(end_col)
range = "Sheet1!A1:" + str(end_col_letter) + "1"
# adding columns
self.sheet_service.spreadsheets().values().update(
spreadsheetId=spreadsheet_id,
range=range,
valueInputOption="RAW",
body=body).execute()
# adding columns to 2nd sheet that can be used for storing data validation ranges (this avoids limitations on number of dropdown items in excel and openoffice)
range = "Sheet2!A1:" + str(end_col_letter) + "1"
self.sheet_service.spreadsheets().values().update(
spreadsheetId=spreadsheet_id,
range=range,
valueInputOption="RAW",
body=body).execute()
# format column header row
header_format_body = {
"requests": [{
"repeatCell": {
"range": {
"startRowIndex": 0,
"endRowIndex": 1
},
"cell": {
"userEnteredFormat": {
"backgroundColor": {
"red": 224.0 / 255,
"green": 224.0 / 255,
"blue": 224.0 / 255
},
"horizontalAlignment": "CENTER",
"textFormat": {
"foregroundColor": {
"red": 0.0 / 255,
"green": 0.0 / 255,
"blue": 0.0 / 255
},
"fontSize": 8,
"bold": True
}
}
},
"fields":
"userEnteredFormat(backgroundColor,textFormat,horizontalAlignment)"
}
}, {
"updateSheetProperties": {
"properties": {
"gridProperties": {
"frozenRowCount": 1
}
},
"fields": "gridProperties.frozenRowCount"
}
}, {
"autoResizeDimensions": {
"dimensions": {
"dimension": "COLUMNS",
"startIndex": 0
}
}
}]
}
response = self.sheet_service.spreadsheets().batchUpdate(
spreadsheetId=spreadsheet_id, body=header_format_body).execute()
# adding additional metadata values if needed
# adding value-constraints from data model as dropdowns
# fix for issue #410
# batch google API request to create metadata template
data = []
for i, req in enumerate(ordered_metadata_fields[0]):
values = required_metadata_fields[req]
if self.additional_metadata and req in self.additional_metadata:
values = self.additional_metadata[req]
target_col_letter = self._column_to_letter(i)
range_vals = target_col_letter + '2:' + target_col_letter + str(
len(values) + 1)
data.append({
"range": range_vals,
"majorDimension": "COLUMNS",
"values": [values]
})
batch_update_values_request_body = {
# How the input data should be interpreted.
"valueInputOption": "RAW",
# The new values to apply to the spreadsheet.
"data": data
}
response = self.sheet_service.spreadsheets().values().batchUpdate(
spreadsheetId=spreadsheet_id,
body=batch_update_values_request_body).execute()
# end of fix for issue #410
#store all requests to execute at once
requests_body = {}
requests_body["requests"] = []
for i, req in enumerate(ordered_metadata_fields[0]):
values = required_metadata_fields[req]
# adding description to headers
# this is not executed if only JSON schema is defined
# TODO: abstract better and document
# also formatting required columns
if self.sg.se:
# get node definition
note = self.sg.get_node_definition(req)
notes_body = {
"requests": [{
"updateCells": {
"range": {
"startRowIndex": 0,
"endRowIndex": 1,
"startColumnIndex": i,
"endColumnIndex": i + 1
},
"rows": [{
"values": [{
"note": note
}]
}],
"fields": "note"
}
}]
}
requests_body["requests"].append(notes_body["requests"])
# get node validation rules if any
validation_rules = self.sg.get_node_validation_rules(req)
# if 'list' in validation rules add a note with instructions on
# adding a list of multiple values
# TODO: add validation and QC rules "compiler/generator" class elsewhere
# for now have the list logic here
if "list" in validation_rules:
note = "From 'Selection options' menu above, go to 'Select multiple values', check all items that apply, and click 'Save selected values'"
notes_body = {
"requests": [{
"repeatCell": {
"range": {
"startRowIndex": 1,
"startColumnIndex": i,
"endColumnIndex": i + 1
},
"cell": {
"note": note
},
"fields": "note"
}
}]
}
requests_body["requests"].append(notes_body["requests"])
# update background colors so that columns that are required are highlighted
# check if attribute is required and set a corresponding color
if req in json_schema["required"]:
bg_color = CONFIG["style"]["google_manifest"].get(
"req_bg_color", {
"red": 0.9215,
"green": 0.9725,
"blue": 0.9803,
})
req_format_body = {
"requests": [{
"repeatCell": {
"range": {
"startColumnIndex": i,
"endColumnIndex": i + 1
},
"cell": {
"userEnteredFormat": {
"backgroundColor": bg_color
}
},
"fields": "userEnteredFormat(backgroundColor)"
}
}]
}
requests_body["requests"].append(req_format_body["requests"])
# adding value-constraints if any
req_vals = [{
"userEnteredValue": value
} for value in values if value]
if not req_vals:
continue
# generating sheet api request to populate a dropdown or a multi selection UI
if len(req_vals) > 0 and not "list" in validation_rules:
# if more than 0 values in dropdown use ONE_OF_RANGE type of validation since excel and openoffice
# do not support other kinds of data validation for larger number of items (even if individual items are not that many
# excel has a total number of characters limit per dropdown...)
validation_body = self._get_column_data_validation_values(
spreadsheet_id,
req_vals,
i,
validation_type="ONE_OF_RANGE")
elif "list" in validation_rules:
# if list is in validation rule attempt to create a multi-value
# selection UI, which requires explicit valid values range in
# the spreadsheet
validation_body = self._get_column_data_validation_values(
spreadsheet_id,
req_vals,
i,
strict=False,
custom_ui=False,
input_message="",
validation_type="ONE_OF_RANGE")
else:
validation_body = self._get_column_data_validation_values(
spreadsheet_id, req_vals, i)
requests_body["requests"].append(validation_body["requests"])
# generate a conditional format rule for each required value (i.e. valid value)
# for this field (i.e. if this field is set to a valid value that may require additional
# fields to be filled in, these additional fields will be formatted in a custom style (e.g. red background)
for req_val in req_vals:
# get this required/valid value's node label in schema, based on display name (i.e. shown to the user in a dropdown to fill in)
req_val = req_val["userEnteredValue"]
req_val_node_label = self.sg.get_node_label(req_val)
if not req_val_node_label:
# if this node is not in the graph
# continue - there are no dependencies for it
continue
# check if this required/valid value has additional dependency attributes
val_dependencies = self.sg.get_node_dependencies(
req_val_node_label, schema_ordered=False)
# prepare request calls
dependency_formatting_body = {"requests": []}
if val_dependencies:
# if there are additional attribute dependencies find the corresponding
# fields that need to be filled in and construct conditional formatting rules
# indicating the dependencies need to be filled in
# set target ranges for this rule
# i.e. dependency attribute columns that will be formatted
# find dependency column indexes
# note that dependencies values must be in index
# TODO: catch value error that shouldn't happen
column_idxs = [
ordered_metadata_fields[0].index(val_dep)
for val_dep in val_dependencies
]
# construct ranges based on dependency column indexes
rule_ranges = self._columns_to_sheet_ranges(column_idxs)
# go over valid value dependencies
for j, val_dep in enumerate(val_dependencies):
is_required = False
if self.sg.is_node_required(val_dep):
is_required = True
else:
is_required = False
# construct formatting rule
formatting_rule = self._column_to_cond_format_eq_rule(
i, req_val, required=is_required)
# construct conditional format rule
conditional_format_rule = {
"addConditionalFormatRule": {
"rule": {
"ranges": rule_ranges[j],
"booleanRule": formatting_rule,
},
"index": 0
}
}
dependency_formatting_body["requests"].append(
conditional_format_rule)
# check if dependency formatting rules have been added and update sheet if so
if dependency_formatting_body["requests"]:
requests_body["requests"].append(
dependency_formatting_body["requests"])
# setting cell borders
cell_range = {
"sheetId": 0,
"startRowIndex": 0,
}
requests_body["requests"].append(self._get_cell_borders(cell_range))
execute_google_api_requests(self.sheet_service,
requests_body,
service_type="batch_update",
spreadsheet_id=spreadsheet_id)
# setting up spreadsheet permissions (setup so that anyone with the link can edit)
self._set_permissions(spreadsheet_id)
# generating spreadsheet URL
manifest_url = "https://docs.google.com/spreadsheets/d/" + spreadsheet_id
# print("========================================================================================================")
# print("Manifest successfully generated from schema!")
# print("URL: " + manifest_url)
# print("========================================================================================================")
return manifest_url
def set_dataframe_by_url(self, manifest_url: str,
manifest_df: pd.DataFrame) -> ps.Spreadsheet:
"""Update Google Sheets using given pandas DataFrame.
Args:
manifest_url (str): Google Sheets URL.
manifest_df (pd.DataFrame): Data frame to "upload".
Returns:
ps.Spreadsheet: A Google Sheet object.
"""
# authorize pygsheets to read from the given URL
gc = ps.authorize(custom_credentials=self.creds)
# open google sheets and extract first sheet
sh = gc.open_by_url(manifest_url)
wb = sh[0]
# The following line sets `valueInputOption = "RAW"` in pygsheets
sh.default_parse = False
# update spreadsheet with given manifest starting at top-left cell
wb.set_dataframe(manifest_df, (1, 1))
# set permissions so that anyone with the link can edit
sh.share("", role="writer", type="anyone")
return sh
def get_dataframe_by_url(self, manifest_url: str) -> pd.DataFrame:
"""Retrieve pandas DataFrame from table in Google Sheets.
Args:
manifest_url (str): Google Sheets URL.
Return:
pd.DataFrame: Data frame corresponding to table in given URL.
"""
# authorize pygsheets to read from the given URL
gc = ps.authorize(custom_credentials=self.creds)
# open google sheets and extract first sheet
sh = gc.open_by_url(manifest_url)
wb = sh[0]
# get column headers and read it into a dataframe
manifest_df = wb.get_as_df(hasHeader=True)
# An empty column is sometimes included
if "" in manifest_df:
manifest_df.drop(columns=[""], inplace=True)
return manifest_df
def map_annotation_names_to_display_names(
self, annotations: pd.DataFrame) -> pd.DataFrame:
"""Update columns names to use display names for consistency.
Args:
annotations (pd.DataFrame): Annotations table.
Returns:
pd.DataFrame: Annotations table with updated column headers.
"""
# Get list of attribute nodes from data model
model_nodes = self.sg.se.get_nx_schema().nodes
# Subset annotations to those appearing as a label in the model
labels = filter(lambda x: x in model_nodes, annotations.columns)
# Generate a dictionary mapping labels to display names
label_map = {l: model_nodes[l]["displayName"] for l in labels}
# Use the above dictionary to rename columns in question
return annotations.rename(columns=label_map)
def get_manifest_with_annotations(
self,
annotations: pd.DataFrame) -> Tuple[ps.Spreadsheet, pd.DataFrame]:
"""Generate manifest, optionally with annotations (if requested).
Args:
annotations (pd.DataFrame): Annotations table (can be empty).
Returns:
Tuple[ps.Spreadsheet, pd.DataFrame]: Both the Google Sheet
URL and the corresponding data frame is returned.
"""
# Map annotation labels to display names to match manifest columns
annotations = self.map_annotation_names_to_display_names(annotations)
# Convert annotations table into dictionary, but maintain order
annotations_dict_raw = annotations.to_dict(into=OrderedDict)
annotations_dict = OrderedDict(
(k, list(v.values())) for k, v in annotations_dict_raw.items())
# Needs to happen before get_empty_manifest() gets called
self.additional_metadata = annotations_dict
# Generate empty manifest using `additional_metadata`
manifest_url = self.get_empty_manifest()
manifest_df = self.get_dataframe_by_url(manifest_url)
# Annotations clashing with manifest attributes are skipped
# during empty manifest generation. For more info, search
# for `additional_metadata` in `self.get_empty_manifest`.
# Hence, the shared columns need to be updated separately.
if self.is_file_based and self.use_annotations:
# This approach assumes that `update_df` returns
# a data frame whose columns are in the same order
manifest_df = update_df(manifest_df, annotations)
manifest_sh = self.set_dataframe_by_url(manifest_url, manifest_df)
manifest_url = manifest_sh.url
return manifest_url, manifest_df
def get_manifest(self,
dataset_id: str = None,
sheet_url: bool = None,
json_schema: str = None):
"""Gets manifest for a given dataset on Synapse.
Args:
dataset_id: Synapse ID of the "dataset" entity on Synapse (for a given center/project).
sheet_url: Determines if googlesheet URL or pandas dataframe should be returned.
Returns:
Googlesheet URL (if sheet_url is True), or pandas dataframe (if sheet_url is False).
"""
# Handle case when no dataset ID is provided
if not dataset_id:
return self.get_empty_manifest(json_schema_filepath=json_schema)
# Otherwise, create manifest using the given dataset
syn_store = SynapseStorage()
# Get manifest file associated with given dataset (if applicable)
syn_id_and_path = syn_store.getDatasetManifest(datasetId=dataset_id)
# Populate empty template with existing manifest
if syn_id_and_path:
# TODO: Update or remove the warning in self.__init__() if
# you change the behavior here based on self.use_annotations
# get synapse ID manifest associated with dataset
manifest_data = syn_store.getDatasetManifest(datasetId=dataset_id,
downloadFile=True)
# If the sheet URL isn't requested, simply return a pandas DataFrame
if not sheet_url:
return pd.read_csv(manifest_data.path)
# get URL of an empty manifest file created based on schema component
empty_manifest_url = self.get_empty_manifest()
# populate empty manifest with content from downloaded/existing manifest
pop_manifest_url = self.populate_manifest_spreadsheet(
manifest_data.path, empty_manifest_url)
return pop_manifest_url
# Generate empty template and optionally fill in with annotations
else:
# Using getDatasetAnnotations() to retrieve file names and subset
# entities to files and folders (ignoring tables/views)
annotations = pd.DataFrame()
if self.is_file_based:
annotations = syn_store.getDatasetAnnotations(dataset_id)
# Subset columns if no interested in user-defined annotations
if self.is_file_based and not self.use_annotations:
annotations = annotations[["Filename", "eTag", "entityId"]]
# Update `additional_metadata` and generate manifest
manifest_url, manifest_df = self.get_manifest_with_annotations(
annotations)
if sheet_url:
return manifest_url
else:
return manifest_df
def populate_manifest_spreadsheet(self, existing_manifest_path,
empty_manifest_url):
"""Creates a google sheet manifest based on existing manifest.
Args:
existing_manifest_path: the location of the manifest containing metadata presently stored
empty_manifest_url: the path to a manifest template to be prepopulated with existing's manifest metadata
"""
# read existing manifest
manifest = pd.read_csv(existing_manifest_path).fillna("")
# sort manifest columns
manifest_fields = manifest.columns.tolist()
manifest_fields = self.sort_manifest_fields(manifest_fields)
manifest = manifest[manifest_fields]
# TODO: Handle scenario when existing manifest does not match new
# manifest template due to changes in the data model
manifest_sh = self.set_dataframe_by_url(empty_manifest_url, manifest)
return manifest_sh.url
def sort_manifest_fields(self, manifest_fields, order="schema"):
# order manifest fields alphabetically (base order)
manifest_fields = sorted(manifest_fields)
if order == "alphabetical":
# if the order is alphabetical ensure that filename is first, if present
if "Filename" in manifest_fields:
manifest_fields.remove("Filename")
manifest_fields.insert(0, "Filename")
# order manifest fields based on schema (schema.org)
if order == "schema":
if self.sg and self.root:
# get display names of dependencies
dependencies_display_names = self.sg.get_node_dependencies(
self.root)
# reorder manifest fields so that root dependencies are first and follow schema order
manifest_fields = sorted(
manifest_fields,
key=lambda x: dependencies_display_names.index(x)
if x in dependencies_display_names else len(manifest_fields
) - 1)
else:
raise ValueError(
f"Provide valid data model path and valid component from data model."
)
# always have entityId as last columnn, if present
if "entityId" in manifest_fields:
manifest_fields.remove("entityId")
manifest_fields.append("entityId")
return manifest_fields