def build_table_mapper_map(self):
logger.info("Looking for tables matching TABLE_MAPPING ")
votable = parse(self.votable_path)
for template_key in self.json_view["MODEL_INSTANCE"][
"TABLE_MAPPING"].keys():
logger.info("Looking for a table matching TABLE_MAPPING %s",
template_key)
name = None
parsed_table = None
for table in votable.iter_tables():
if template_key == table.ID:
logger.info("Table with ID = %s found", template_key)
name = table.ID
parsed_table = table
break
if name == None:
for table in votable.iter_tables():
if template_key == table.name:
logger.info("Table with name = %s found", template_key)
name = table.name
parsed_table = table
break
if name == None:
raise Exception("Cannot find table with name or ID = " + name)
else:
logger.info("Add TableMapper for table %s", name)
self.table_mappers[template_key] = TableMapper(
template_key,
self.votable_path,
parsed_table=parsed_table,
json_inst_dict=self.json_view)
def build_json_mapping(self):
logger.info("Formating the JSON view")
builder = JsonMappingBuilder(json_dict=self.json_view)
builder.revert_compositions("COLLECTION")
builder.revert_templates()
builder.revert_elements("INSTANCE")
builder.revert_elements("ATTRIBUTE")
self.json_view = builder.json
def get_flatten_data_head(self, data_subset=None):
if len(self.table_iterators) > 0 :
for key, value in self.table_iterators.items():
if data_subset is None or data_subset == key:
return value._get_flatten_data_head()
raise Exception("cannot find data subset " + str(data_subset))
else:
logger.info("No data table")
return {}
def connect_join_iterators(self):
parse_tables = {}
for template,table_mapper in self.table_mappers.items():
parse_tables[template] = table_mapper.parsed_table
for template,table_mapper in self.table_mappers.items():
for target,join_iterator in table_mapper.join_iterators.items():
logger.info("join template %s with template %s", template, target)
join_iterator.connect_votable(parse_tables[target])
def revert_sets_xxxx(self, name, default_key=None):
logger.info("reverting sets %s", name)
root_element = self.json['VODML']
while True:
self.retour = None
self._revert_set(root_element, name, default_key)
if self.retour is not None:
self.retour["node"].pop(name)
for k, v in self.retour["newcontent"].items():
self.retour["node"][k] = v
else:
break
def revert_elements(self, name, dmrole=None, dmtype=None):
logger.info("reverting elements %s - ('%s':{role ...} -> 'role':{})", name, name)
root_element = self.json['VODML']
while True:
self.retour = None
self._revert_subelement(root_element, name, dmrole, dmtype)
if self.retour is not None:
self.retour["node"].pop(name)
for k, v in self.retour["newcontent"].items():
self.retour["node"][k] = v
else:
break
def _set_value(self, element, role=None, parent_role=None):
"""
Create a column mapping entry for the element if it is a @ref
both role an parent_role are just labels used make more explicit
the string representation of the columns mapping
"""
keys = element.keys()
if ("@dmtype" in keys and "@ref" in keys
and "@value" in keys and element["@value"] == ""):
logger.info("Give role %s to the column %s "
, parent_role, element["@ref"])
self.column_mapping.add_entry(element["@ref"], role, parent_role)
element["@value"] = "array coucou"
def revert_array_2bethrown(self):
# Give TABLE_ROW_TEMPLATEs the same role as its container
logger.info("reverting TABLE_ROW_TEMPLATEs")
root_element = self.json['VODML']
while True:
self.retour = None
self._revert_set(root_element, 'TABLE_ROW_TEMPLATE', None)
if self.retour is not None:
print("============== ")
array_container = self.retour["node"]
array_node = array_container['TABLE_ROW_TEMPLATE']
array_role = array_node['INSTANCE']["@dmrole"]
array_container['TABLE_ROW_TEMPLATE'] = {array_role: [array_node['INSTANCE']]}
else:
break
def _resolve_header_value(self, element):
"""
Set the @value of element with the value of the <PARAM> having
either a ID or a name matching @ref
"""
keys = element.keys()
if ("@dmtype" in keys and "@ref" in keys
and "@value" in keys and element["@value"] == ""):
for param in self.parsed_table.params:
if param.ID == element["@ref"]:
logger.info("set element %s with value=%s of PARAM(ID=%s)"
, str(element), param.value, element["@ref"])
element["@value"] = param.value.decode("utf-8")
elif param.name == element["@ref"] :
logger.info("set element%s with value=%s of PARAM(name=%s)"
, str(element), param.value, element["@ref"])
element["@value"] = param.value.decode("utf-8")
def revert_templates(self):
logger.info("reverting templates - {TEMPLATES:[{ref_table ...} ] -> 'ref_table':{...}")
self.revert_elements("TEMPLATES")
root_element = self.json['VODML']
templates = {}
keys = []
for k, v in root_element.items():
if k not in ['MODELS', 'GLOBALS']:
templates[k] = v
keys.append(k)
logger.info("Put all TEMPLATES{} in a global TEMPLATES[]")
for k in keys:
root_element.pop(k)
root_element["TEMPLATES"] = templates
def resolve_refs_and_values(self, resolve_refs=False):
root_element = self.json['VODML']['TEMPLATES'][self.table_name]
if resolve_refs is True:
logger.info("Replace object references with referenced object copies")
self.resolve_object_references()
logger.info("Resolve references to PARAMS")
self._set_header_values(root_element)
logger.info("Set array iterators")
self._set_array_iterators(root_element)
logger.info("Resolve mapping leaves is an TABLE_ROW_TEMPLATE block")
self._set_array_subelement_values(self.array, parent_role=None)
def revert_compositions(self, name, dmrole=None, dmtype=None):
logger.info("reverting compositions %s - ('%s':[{role ...} ...] -> 'role':[...])", name, name)
root_element = self.json['VODML']
while True:
self.retour = None
self._revert_composition(root_element, name, dmrole, dmtype)
#print("==========")
#print(DictUtils.get_pretty_json(self.retour))
#print("==========")
#sys.exit()
if self.retour is not None:
self.retour["node"].pop(name)
for ele in self.retour["newcontent"]:
for k, v in ele.items():
self.retour["node"][k] = v
else:
break
def revert_templates(self):
logger.info(
"reverting templates - {TABLE_MAPPING:[{ref_table ...} ] -> 'ref_table':{...}"
)
self.revert_elements("TABLE_MAPPING")
root_element = self.json['MODEL_INSTANCE']
templates = {}
keys = []
for k, v in root_element.items():
if k not in ['MODELS', 'GLOBALS']:
templates[k] = v
keys.append(k)
logger.info("Put all TABLE_MAPPING{} in a global TABLE_MAPPING[]")
for k in keys:
root_element.pop(k)
root_element["TABLE_MAPPING"] = templates
def get_full_instance(self, resolve_refs=False):
if resolve_refs is True:
logger.info("Replace object references with referenced object copies")
self.resolve_object_references()
retour = deepcopy(self.json['VODML']['TEMPLATES'][self.table_name])
json_block_extractor = JsonBlockExtractor(retour)
json_block_extractor.search_array_container()
if len(json_block_extractor.searched_elements) > 0 :
json_block_extractor.searched_elements[0][0] = {}
cpt = 0
inst = None
while True:
inst = self._get_next_row_instance()
if inst is None:
break
elif cpt == 0:
json_block_extractor.searched_elements[0][0] = inst
else:
json_block_extractor.searched_elements[0].append(inst)
cpt += 1
return retour
def build_json_view(self):
logger.info("Extracting the MODEL_INSTANCE block")
instanceFromVotable = InstanceFromVotable(self.votable_path)
instanceFromVotable._extract_vodml_block()
logger.info("Validating the MODEL_INSTANCE block")
instanceFromVotable._validate_vodml_block()
logger.info("Extracting the raw JSON block")
self.json_view = instanceFromVotable.json_block
def __init__(self,
table_name,
votable_path,
parsed_table=None,
json_inst_path=None ,
json_inst_dict=None):
'''
Constructor
'''
self.table_name = table_name
self.parsed_table_path = votable_path
if parsed_table is None:
logger.info("take the first table")
self.parsed_table = parse_single_table(self.parsed_table_path)
else:
logger.info("take the given parsed table")
self.parsed_table = parsed_table
self.table = self.parsed_table.to_table()
if json_inst_path is not None:
self.json_path = json_inst_path
with open(self.json_path) as json_file:
self.json = json.load(json_file)
else:
self.json = json_inst_dict
self.json_path = None
# array block reference
self.array = None
# key = role of the instance contained TABLE_ROW_TEMPLATE value = TableIterator
self.table_iterators = {}
self.column_mapping = ColumnMapping()
# key = foreign table name value = TableIterator
self.join_iterators = {}
self.join = None
def _get_next_row_instance(self, data_subset=None):
if len(self.table_iterators) > 0 :
# One table_iterator par TABLE_ROW_TEMPLATE block
# For now, the case with multiple table_iterator has not been tested
for key, table_iterator in self.table_iterators.items():
if data_subset is None or data_subset == key:
next_row_instance = table_iterator._get_next_row_instance()
if next_row_instance is None:
return None
for _, join_iterator in self.join_iterators.items():
primary_column = self.column_mapping.get_col_index_by_name(join_iterator.primary_key)
join_iterator.set_foreignkey_value(table_iterator.last_row[primary_column])
json_block_extractor = JsonBlockExtractor(next_row_instance)
json_block_extractor.search_join_container()
if len(json_block_extractor.searched_elements) > 0 :
json_block_extractor.searched_elements[0][0] = {}
cpt = 0
inst = None
while True:
inst = join_iterator.table_mapper._get_next_row_instance()
if inst is None:
break
elif cpt == 0:
json_block_extractor.searched_elements[0][0] = inst
else:
json_block_extractor.searched_elements[0].append(inst)
cpt += 1
return next_row_instance
raise Exception("cannot find data subset " + data_subset)
else:
logger.info("No data table")
return {}
def _set_array_iterators(self, root_element):
"""
Build and array iterator for each TABLE_ROW_TEMPLATE element found within root_element
"""
if isinstance(root_element, list):
for idx, _ in enumerate(root_element):
if self.retour is None:
self._set_array_iterators(root_element[idx])
elif isinstance(root_element, dict):
for k, v in root_element.items():
if k == 'TABLE_ROW_TEMPLATE':
self.array = v
row_filter = None
iterator_key = None
for ro in self.array.keys():
if ro == 'FILTER' :
row_filter = RowFilter(self.array['FILTER'])
iterator_key = row_filter.name
logger.info("Add filter %s", row_filter.name)
else:
iterator_key = ro
logger.info("Set table iterator for object with role=%s", ro)
self.table_iterators[ro] = TableIterator(
iterator_key,
self.parsed_table.to_table(),
self.array[ro],
self.column_mapping,
row_filter=row_filter
)
pass
elif k == 'JOIN':
self.join = v
iterator_key = self.join["@table_ref"]
logger.info("Set join iterator with table=%s", iterator_key)
for ro in self.join.keys():
self.join_iterators[iterator_key] = JoinIterator(
iterator_key,
self.join["@primary"],
self.join["@foreign"],
self.join
)
# JOIN has only one child
break
pass
if isinstance(v, list):
for ele in v:
self._set_array_iterators(ele)
elif isinstance(v, dict):
#self._set_value(v)
self._set_array_iterators(v)
def get_root_element(self, root_class):
for template,table_mapper in self.table_mappers.items():
logger.info("Looking for %s instances in template %s", root_class, template)
json_block_extract = JsonBlockExtractor(table_mapper.json)
retour = json_block_extract.search_subelement_by_type(root_class)
for block in retour:
if "@dmrole" not in block.keys():
logger.info("found (no role)")
return table_mapper
role = block["@dmrole"]
if role == "" or role == "root":
logger.info("found with role=%s", role)
return table_mapper
return None
def populate_templates(self, resolve_refs=False):
for k, v in self.table_mappers.items():
logger.info("populate template %s", k)
v.resolve_refs_and_values(resolve_refs=resolve_refs)
v.map_columns()
def save_instance(self):
file_path = self.json_path.replace(".json", ".inst.json")
logger.info("save instance in %s", file_path)
with open(file_path, 'w') as jsonfile:
jsonfile.write(DictUtils.get_pretty_json(self.json))
def _revert_subelement(self, root_element, name, dmrole, dmtype):
if isinstance(root_element, list):
for idx, _ in enumerate(root_element):
if self.retour is None:
self._revert_subelement(root_element[idx], name, dmrole, dmtype)
elif isinstance(root_element, dict):
for k, v in root_element.items():
if k == name:
if isinstance(v, list):
#print(DictUtils.get_pretty_json(v))
'''
newcontent = {}
ele_array = []
for ele in v:
new_key = self._get_key_for_element(ele)
new_ele = deepcopy(ele)
self._drop_role_if_needed(new_ele)
self._add_value_if_needed(new_ele)
ele_array.append(new_ele)
newcontent[new_key] = deepcopy(ele_array)
'''
# if we got an array of objects with all the same role
# we have a composition of instances. In that case that
# role is given the composition object and to object array
# is given as the composition content
former_key = ""
is_array = True
for ele in v:
new_key = self._get_key_for_element(ele)
if former_key == "":
former_key = new_key
if former_key != new_key:
is_array = False
break
if is_array is False:
newcontent = {}
for ele in v:
new_key = self._get_key_for_element(ele)
new_ele = deepcopy(ele)
self._drop_role_if_needed(new_ele)
self._add_value_if_needed(new_ele)
newcontent[new_key] = new_ele
else:
logger.info("find a composition of %s with the role=%s", name, former_key)
newcontent = {}
new_array = []
for ele in v:
new_ele = deepcopy(ele)
#self._drop_role_if_needed(new_ele)
self._add_value_if_needed(new_ele)
new_array.append(new_ele)
newcontent[former_key] = new_array
self.retour = {'node': root_element, "newcontent": newcontent}
elif isinstance(v, dict):
newcontent = {}
new_key = self._get_key_for_element(v)
newcontent[new_key] = deepcopy(v)
self._add_value_if_needed(newcontent[new_key])
self._drop_role_if_needed(newcontent[new_key])
self.retour = {'node': root_element, "newcontent": newcontent}
if self.retour is None:
self._revert_subelement(v, name, dmrole, dmtype)
