def get_data_element(existing_dataset, data_element_payload, distribution_name,
auth, aristotleurl, verbose):
"""
Get data element id. Either from existing dataset or by creating new
"""
logical_path = data_element_payload['logical_path']
if (distribution_name in existing_dataset and
logical_path in existing_dataset[distribution_name]['tables']):
data_element_id = (
existing_dataset[distribution_name]['tables'][logical_path]
)
else:
value_domain_payload = (
data_element_payload['data_element']['fields']['valueDomain']
)
value_domain_id = get_value_domain(
value_domain_payload,
auth,
aristotleurl,
verbose)
data_element_payload['data_element']['fields']['valueDomain'] = (
value_domain_id
)
data_element_id = utils.request_post(
auth=auth,
payload=data_element_payload['data_element'],
url=aristotleurl,
verbose=verbose
)
return data_element_id
def execute_saved_req(auth, file, dbuuid, aristotleurl, verbose):
"""
This script consumes the json file to upload metadata to Aristotle
"""
json_data = utils.read_file(file)
existing_dataset = {}
if dbuuid:
try:
existing_dataset = create_dataset_structure(
dbuuid,
auth,
aristotleurl,
verbose)
except ValueError as err:
print(err)
return
for distribution in json_data:
distribution_name = distribution['fields']['name']
if distribution_name in existing_dataset:
# This updates the distribution
distribution['fields']['uuid'] = (
existing_dataset[distribution_name]['uuid']
)
data_elements = distribution['fields']['data_elements']
for i, data_element_payload in enumerate(data_elements):
data_elements[i]['data_element'] = get_data_element(
existing_dataset,
data_element_payload,
distribution_name,
auth,
aristotleurl,
verbose)
utils.request_post(
auth=auth,
payload=distribution,
url=aristotleurl,
verbose=verbose
)
def test_request_post(self):
responses.add(
responses.POST,
'http://127.0.0.1:8080/api/v3_1/metadata/',
json={'created':[{'uuid':'uuid1'}]},
status=200,
content_type='application/json'
)
result = utils.request_post(
"testAuth",
payload={'dataelement':'test', "fields":{}},
other_field_data={'valuedomain':'testvaluedomain'},
verbose=True
)
self.assertEqual('uuid1',result)
def miner(url, database, auth, file, aristotleurl, verbose):
"""
This script creates a data.json file,
that contains all the database schema to be uploaded in Aristotle
"""
engine = create_engine(url)
metadata = MetaData()
conn = engine.connect()
metadata.reflect(engine)
table_data = {}
distributions = []
dataset = utils.create_req(model="dataset",
name=database,
app="aristotle_dse")
dataset = utils.request_post(auth=auth,
payload=dataset,
url=aristotleurl,
verbose=verbose)
for table_object in metadata.sorted_tables:
table = table_object.name
extra_information_distribution = {
"data_elements": [],
"dataset": dataset
}
table_data[table] = []
for columns in metadata.tables[table].c:
value_domain = create_value_domain_request(columns)
data_element = create_data_element_request(columns, value_domain)
extra_information_distribution['data_elements'].append({
'data_element':
data_element,
"logical_path":
table + "." + str(columns.name)
})
distribution = create_distribution_request(
table_object, extra_information_distribution)
distributions.append(distribution)
utils.save_req_file(distributions, file)
conn.close()
print(dataset)
def get_value_domain(value_domain_payload, auth, aristotleurl, verbose):
"""
Get value domain id. Either from existing or by creating new
"""
valueDomainResponse = utils.request_get(
auth=auth,
payload={
'name__icontains': value_domain_payload['fields']['name'],
'type': 'aristotle_mdr:valuedomain'
},
url=aristotleurl)
if valueDomainResponse.json()['count'] > 0:
value_domain_id = valueDomainResponse.json()['results'][0]['uuid']
else:
value_domain_id = False
if not value_domain_id:
value_domain_id = utils.request_post(
auth=auth,
payload=value_domain_payload,
url=aristotleurl,
verbose=verbose
)
return value_domain_id