def create_dataset(connector: NomisApiConnector,
transformations: DatasetTransformations) -> None:
"""
Initialise a dataset using the jsonstat table either read in or retrieved from Cantabular.
:param connector: An open, initialised instance of `NomisApiConnector`.
:param transformations: An initialised instance of `DatasetTransformations` with a valid table attribute.
"""
connector.create_dataset(
args.dataset_id,
transformations.dataset_creation(args.dataset_id, args.dataset_title))
def handle_observations(
connector: NomisApiConnector,
transformations: DatasetTransformations,
) -> None:
"""
Append/overwrite observations to the dataset.
:param connector: An open, initialised instance of `NomisApiConnector`.
:param transformations: An initialised instance of `DatasetTransformations` with a valid table attribute.
"""
logger.debug("\n-----APPENDING OBSERVATIONS-----")
connector.overwrite_dataset_observations(
args.dataset_id, transformations.observations(args.dataset_id))
def cantabular_metadata(file_data: dict) -> List[UuidMetadata]:
"""
Function for handling metadata in the Cantabular format.
:param file_data: A dictionary containing a Python dict representation of the inputted metadata file.
:return: A list of namedtuples containing a UUID (str) and its associated metadata (dict).
"""
logger.info("Handling metadata in the Cantabular format.")
# Extract variable IDs
data = [meta["meta"] for meta in file_data["vars"]]
variables = [meta["id"] for meta in file_data["vars"]]
uuids_metadata: List[UuidMetadata] = []
# Get UUIDs from Nomis
with NomisApiConnector(config.get_credentials('nomis'),
config.get_client('nomis')) as connector:
for variable in range(0, len(variables)):
response = connector.get_variable(variables[variable],
return_bool=True)
if response is not False:
uuids_metadata.append(
UuidMetadata(response["uuid"], data[variable]))
return uuids_metadata
def handle_dimensions(
connector: NomisApiConnector,
transformations: DatasetTransformations,
key: Union[str, None],
) -> None:
"""
Assign dimensions to the dataset.
:param connector: An open, initialised instance of `NomisApiConnector`.
:param transformations: An initialised instance of `DatasetTransformations` with a valid table attribute.
:param key: Key value for dimensions.
"""
logger.debug("\n-----ASSIGNING DIMENSIONS-----")
connector.assign_dimensions_to_dataset(
args.dataset_id, transformations.assign_dimensions(key))
def check_dataset_dimensions(connector: NomisApiConnector,
dimensions: list) -> bool:
"""
Obtain a list of all variables marked for posting that haven't already been assigned to the dataset.
:param connector: An open, initialised instance of `NomisApiConnector`.
:param dimensions: List of dimensions.
:return: A list of variables (from the arguments) that have not yet been assigned to the dataset.
"""
assigned_variables_json = connector.get_dataset_dimensions(args.dataset_id)
if isinstance(assigned_variables_json, list):
assigned_variables = [
assigned_variables_json[i]['name']
for i in range(len(assigned_variables_json))
]
else:
raise TypeError(
"Unexpected type received from the Nomis API Connector.")
if "geography" in assigned_variables:
assigned_variables.remove("geography")
if dimensions == assigned_variables:
return True
return False
def data_main() -> None:
"""
Main function for handling datasets.
"""
logger.info(f"Commencing data transformation service.")
with NomisApiConnector(config.get_credentials('nomis'),
config.get_client('nomis')) as connector:
exists = check_dataset_exists(connector)
dataset_transformations(connector, exists, retrieve_data())
logger.info(
f"DATA TRANSFORMATION SUCCESS: A dataset with the ID {args.dataset_id} has been "
f"{'UPDATED' if exists else 'CREATED'} successfully.")
def ons_metadata(file_data: dict) -> List[UuidMetadata]:
"""
Function for handling metadata in the ONS format.
:param file_data: A dictionary containing a Python dict representation of the inputted metadata file.
:return: A list of namedtuples containing a UUID (str) and its associated metadata (dict).
"""
logger.info("Handling metadata in the ONS format.")
# Get a list of all UUIDs in the Nomis system
with NomisApiConnector(config.get_credentials('nomis'),
config.get_client('nomis')) as connector:
nomis_uuids = [
variable["uuid"] for variable in connector.get_variable()
]
uuids_metadata: List[UuidMetadata] = []
# Extract UUIDs and metadata
for meta in file_data["dataModel"]["dataTypes"]:
# Check if the UUIDs match those in the Nomis system; otherwise, skip
if meta["id"] in nomis_uuids:
uuids_metadata.append(
UuidMetadata(meta["id"], {"description": meta["description"]}))
else:
logger.debug(
f"{meta['label']} with UUID {meta['id']} does not exist in NOMIS system"
)
for meta_x in file_data["dataModel"]["childDataClasses"]:
# Check other variable branches
for meta_y in meta_x["childDataElements"]:
# Check if the UUIDs match those in the Nomis system; otherwise, skip
if meta_y["id"] in nomis_uuids:
uuids_metadata.append(
UuidMetadata(meta_y["id"],
{"description": meta_y["description"]}))
else:
logger.debug(
f"VARIABLE: '{meta_y['label']}' with UUID: '{meta_y['id']}' does not exist in NOMIS system"
)
return uuids_metadata
def handle_variables(connector: NomisApiConnector,
transformations: DatasetTransformations,
variables: List[str]) -> None:
"""
Handle variable transmission/manipulation.
:param connector: An open, initialised instance of `NomisApiConnector`.
:param transformations: An initialised instance of `DatasetTransformations` with a valid table attribute.
:param variables: A list of variables to be assigned to the dataset.
"""
logger.debug("\n-----VARIABLE CREATION-----")
# Create the variable creation and category request bodies
variable_request_body = transformations.variable_creation()
type_request_body = transformations.type_creation()
type_ids = get_type_ids(type_request_body)
category_request_body = transformations.category_creation(type_ids)
for variable in variables:
# Check to see variables already exist for the given dimensions; IF the variable does NOT exist then create it
if not connector.get_variable(variable, return_bool=True):
# Create variable
for request in variable_request_body:
if request["name"] == variable:
connector.create_variable(variable, request)
# Create variable type
requests = []
for request in type_request_body:
if request["reference"] == variable:
requests.append(request)
connector.create_variable_type(variable, requests)
# Create the categories for this new variable
requests = []
for category in transformations.table["dimension"][variable][
"category"]["index"]:
for request in category_request_body:
if category == request["code"] and transformations \
.table["dimension"][variable]["category"]["label"][category] == request["title"]:
requests.append(request)
connector.create_variable_category(variable, requests)
def check_dataset_exists(connector: NomisApiConnector) -> bool:
"""
Check whether the dataset already exists in the Nomis database, handle appropriately.
:param connector: An open, initialised instance of `NomisApiConnector`.
:return: A bool indicating if the dataset does exist (`True`) or it doesn't exist (`False`).
"""
exists = connector.get_dataset(args.dataset_id, return_bool=True)
if exists and not args.suppress_prompts:
print(
f"A DATASET WITH ID {args.dataset_id} ALREADY EXISTS. DO YOU WANT TO UPDATE IT? y/n"
)
while 1:
answer = input()
if answer.lower() == 'n':
print("DATASET HAS NOT BEEN UPDATED")
sys.exit(0)
elif answer.lower() == 'y':
break
print("PLEASE INPUT EITHER 'y' (YES) OR 'n' (NO)")
if isinstance(exists, bool):
return exists
else:
raise TypeError("Received unexpected type from Nomis API Connector.")